435 lines
15 KiB
Python
435 lines
15 KiB
Python
"""Crawl4AI 异步新闻抓取引擎
|
||
|
||
海外服务器运行，约束：内存 ≤ 2GB，串行抓取，单源 ≤ 2 小时。
|
||
所有业务参数从 configs/system.yaml 的 crawler 节读取。
|
||
"""
|
||
|
||
import asyncio
|
||
import gc
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from urllib.parse import urljoin
|
||
|
||
import psutil
|
||
import yaml
|
||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
|
||
|
||
from crawler.models import ArticleItem, CrawlResult, SourceConfig
|
||
from crawler.utils import get_news_day
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ── 从配置文件加载常量 ──────────────────────────────────
|
||
|
||
|
||
def _load_crawler_config() -> dict:
|
||
"""从 system.yaml 读取 crawler 配置节（含 Profile 覆盖）"""
|
||
try:
|
||
from crawler.config import load_system_config
|
||
cfg = load_system_config()
|
||
return cfg.get("crawler", {})
|
||
except Exception:
|
||
logger.warning("读取 system.yaml 失败，使用默认值")
|
||
return {}
|
||
|
||
_cfg = _load_crawler_config()
|
||
|
||
MAX_MEMORY_MB = int(_cfg.get("max_memory_mb", 1800))
|
||
ARTICLE_DELAY_SEC = float(_cfg.get("article_delay_sec", 3.0))
|
||
PAGE_TIMEOUT_MS = int(_cfg.get("page_timeout_sec", 45)) * 1000
|
||
SOURCE_TIMEOUT_SEC = int(_cfg.get("source_timeout_sec", 7200))
|
||
_VIEWPORT_WIDTH = int(_cfg.get("viewport_width", 1024))
|
||
_VIEWPORT_HEIGHT = int(_cfg.get("viewport_height", 768))
|
||
_USER_AGENT = _cfg.get(
|
||
"user_agent",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/131.0.0.0 Safari/537.36",
|
||
)
|
||
_HEADFUL = bool(_cfg.get("headful", False))
|
||
_XVFB_DISPLAY = _cfg.get("xvfb_display", ":99")
|
||
|
||
# ── 代理配置 ──────────────────────────────────────────
|
||
|
||
|
||
def _load_proxy_config() -> dict:
|
||
"""从 system.yaml 读取 proxy 配置节（含 Profile 覆盖）"""
|
||
try:
|
||
from crawler.config import load_system_config
|
||
cfg = load_system_config()
|
||
return cfg.get("proxy", {})
|
||
except Exception:
|
||
pass
|
||
return {}
|
||
|
||
_proxy_cfg = _load_proxy_config()
|
||
PROXY_ENABLED = bool(_proxy_cfg.get("enabled", False))
|
||
PROXY_URL = _proxy_cfg.get("url", "socks5://127.0.0.1:1080")
|
||
|
||
# ── 工具函数 ──────────────────────────────────────────
|
||
|
||
|
||
def compute_url_hash(url: str) -> str:
|
||
"""计算 URL 的 SHA256 前 16 个字符作为短文件名"""
|
||
return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
|
||
|
||
|
||
def _get_memory_mb() -> float:
|
||
"""获取当前进程内存使用量（MB）"""
|
||
proc = psutil.Process(os.getpid())
|
||
return proc.memory_info().rss / (1024 * 1024)
|
||
|
||
|
||
def _check_memory(source_id: str) -> None:
|
||
"""检查内存，超限时警告"""
|
||
mem = _get_memory_mb()
|
||
if mem > MAX_MEMORY_MB:
|
||
logger.warning("[%s] ⚠️ 内存使用 %.0f MB 超过上限 %.0f MB，触发 GC",
|
||
source_id, mem, MAX_MEMORY_MB)
|
||
gc.collect()
|
||
mem_after = _get_memory_mb()
|
||
logger.info("[%s] GC 后内存: %.0f MB (回收 %.0f MB)",
|
||
source_id, mem_after, mem - mem_after)
|
||
|
||
# ── 浏览器配置 ──────────────────────────────────────────
|
||
|
||
|
||
def _make_browser_config(source: SourceConfig) -> BrowserConfig:
|
||
"""浏览器配置：根据 anti_bot_mode 和系统配置选择策略
|
||
|
||
- None: 标准轻量 headless（默认）
|
||
- "stealth": 反检测 headless（隐藏自动化特征）
|
||
- "headful": 非 headless 模式（有头浏览器，最像真人）
|
||
|
||
代理：如果 system.yaml proxy.enabled=true，浏览器流量走 SOCKS5 代理。
|
||
headful 模式自动设置 DISPLAY 环境变量（支持 xvfb）。
|
||
"""
|
||
mode = source.anti_bot_mode
|
||
# headless 判定：
|
||
# - 系统级 _HEADFUL=true → 强制有头（覆盖所有 source 模式）
|
||
# - source mode="headful" → 有头
|
||
# - 其他 → headless
|
||
if _HEADFUL:
|
||
headless = False # 系统级强制 headful
|
||
elif mode == "headful":
|
||
headless = False
|
||
else:
|
||
headless = True
|
||
|
||
# 基础反检测参数
|
||
extra_args = [
|
||
"--disable-dev-shm-usage",
|
||
"--disable-gpu",
|
||
"--no-sandbox",
|
||
]
|
||
|
||
# 反检测参数：stealth 模式或系统级 headful 都启用
|
||
if mode == "stealth" or _HEADFUL:
|
||
extra_args += [
|
||
"--disable-blink-features=AutomationControlled",
|
||
"--disable-features=IsolateOrigins,site-per-process",
|
||
]
|
||
|
||
# ── SOCKS5 代理 ──────────────────────────────────
|
||
if PROXY_ENABLED and PROXY_URL:
|
||
extra_args.append(f"--proxy-server={PROXY_URL}")
|
||
logger.info("🌐 浏览器代理已启用: %s", PROXY_URL)
|
||
|
||
# ── headful 模式：设置虚拟显示器 ──────────────────
|
||
if not headless:
|
||
display = os.environ.get("DISPLAY", "")
|
||
if not display:
|
||
os.environ["DISPLAY"] = _XVFB_DISPLAY
|
||
logger.info("🖥️ headful 模式: DISPLAY=%s", _XVFB_DISPLAY)
|
||
|
||
return BrowserConfig(
|
||
browser_type="chromium",
|
||
headless=headless,
|
||
viewport_width=_VIEWPORT_WIDTH,
|
||
viewport_height=_VIEWPORT_HEIGHT,
|
||
verbose=False,
|
||
text_mode=headless, # headful 模式下不禁用图片（更像真人）
|
||
light_mode=headless,
|
||
user_agent=_USER_AGENT,
|
||
extra_args=extra_args,
|
||
)
|
||
|
||
|
||
def _make_run_config(source: SourceConfig) -> CrawlerRunConfig:
|
||
"""抓取运行时配置"""
|
||
return CrawlerRunConfig(
|
||
cache_mode=CacheMode.BYPASS,
|
||
page_timeout=PAGE_TIMEOUT_MS,
|
||
wait_until="domcontentloaded",
|
||
scan_full_page=False,
|
||
simulate_user=True,
|
||
override_navigator=True,
|
||
)
|
||
|
||
def _extract_domain(url: str) -> str:
|
||
"""从 URL 提取域名（去掉 www 前缀）。"""
|
||
from urllib.parse import urlparse
|
||
host = urlparse(url).hostname or ""
|
||
return host.removeprefix("www.").lower()
|
||
|
||
|
||
# ── 链接提取 ──────────────────────────────────────────
|
||
|
||
|
||
async def _extract_article_urls(
|
||
crawler: AsyncWebCrawler,
|
||
source: SourceConfig,
|
||
) -> list[str]:
|
||
"""从首页抓取符合 article_url_pattern 的文章链接（增量：跳过已抓取过的 URL）"""
|
||
config = _make_run_config(source)
|
||
|
||
try:
|
||
result = await crawler.arun(url=source.homepage, config=config)
|
||
except Exception as e:
|
||
logger.error("抓取首页失败 [%s] %s: %s", source.id, source.homepage, e)
|
||
return []
|
||
|
||
if not result.success:
|
||
logger.error("首页返回失败 [%s]: %s", source.id, result.error_message)
|
||
return []
|
||
|
||
html = result.html or ""
|
||
pattern = re.compile(r'href=["\']([^"\']*?)["\']', re.IGNORECASE)
|
||
raw_urls: list[str] = pattern.findall(html)
|
||
|
||
article_url_pattern = re.compile(source.article_url_pattern, re.IGNORECASE)
|
||
base_without_fragment = source.homepage.split("#")[0]
|
||
base_domain = _extract_domain(source.homepage)
|
||
|
||
# 非内容类扩展名
|
||
_SKIP_EXT = re.compile(
|
||
r"\.(png|ico|gif|jpg|jpeg|svg|webp|css|js|xml|json|rss|pdf|zip|woff2?|ttf|eot)"
|
||
r"([?#]|$)",
|
||
re.IGNORECASE,
|
||
)
|
||
# 非文章技术路径
|
||
_SKIP_PATH = re.compile(
|
||
r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)"
|
||
r"[/.]",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
article_urls: list[str] = []
|
||
for raw in raw_urls:
|
||
raw_stripped = raw.strip()
|
||
if raw_stripped.startswith("#") or not raw_stripped:
|
||
continue
|
||
if raw_stripped.lower().startswith("javascript:"):
|
||
continue
|
||
|
||
full = urljoin(source.homepage, raw_stripped)
|
||
full_clean = full.split("#")[0]
|
||
|
||
# 过滤非内容类 URL
|
||
if _SKIP_EXT.search(full_clean):
|
||
continue
|
||
if _SKIP_PATH.search(full_clean):
|
||
continue
|
||
# 过滤非同域链接（广告、CDN 等）
|
||
if _extract_domain(full_clean) != base_domain:
|
||
continue
|
||
|
||
if full_clean.rstrip("/") == base_without_fragment.rstrip("/"):
|
||
continue
|
||
if article_url_pattern.search(full_clean):
|
||
article_urls.append(full_clean)
|
||
|
||
# 本批去重
|
||
seen: set[str] = set()
|
||
unique: list[str] = []
|
||
for u in article_urls:
|
||
if u not in seen:
|
||
seen.add(u)
|
||
unique.append(u)
|
||
|
||
# 增量过滤：读取已抓取的 url_hash，跳过已存在 URL
|
||
today = get_news_day()
|
||
index_path = Path(f"data/raw/{source.id}/{today}/index.jsonl")
|
||
existing_hashes: set[str] = set()
|
||
if index_path.exists():
|
||
with open(index_path, encoding="utf-8") as f:
|
||
for line in f:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
item = json.loads(line)
|
||
existing_hashes.add(item.get("url_hash", ""))
|
||
except json.JSONDecodeError:
|
||
continue
|
||
|
||
filtered: list[str] = []
|
||
skipped = 0
|
||
for u in unique:
|
||
uh = compute_url_hash(u)
|
||
if uh in existing_hashes:
|
||
skipped += 1
|
||
logger.debug("[%s] 跳过已抓取: %s", source.id, u[:80])
|
||
continue
|
||
filtered.append(u)
|
||
if len(filtered) >= source.max_articles_per_run:
|
||
break
|
||
|
||
if skipped > 0:
|
||
logger.info("[%s] 增量过滤: 跳过 %d 篇已抓取，剩余 %d 篇待抓取",
|
||
source.id, skipped, len(filtered))
|
||
|
||
logger.info("[%s] 从首页提取到 %d 个文章链接（去重后 %d，增量后 %d）",
|
||
source.id, len(article_urls), len(unique), len(filtered))
|
||
return filtered
|
||
|
||
# ── 单篇抓取 ──────────────────────────────────────────
|
||
|
||
|
||
async def _crawl_single_article(
|
||
crawler: AsyncWebCrawler,
|
||
url: str,
|
||
source: SourceConfig,
|
||
index: int,
|
||
total: int,
|
||
) -> ArticleItem:
|
||
"""抓取单篇文章（串行模式）"""
|
||
url_hash = compute_url_hash(url)
|
||
now = datetime.now()
|
||
today = get_news_day()
|
||
|
||
article = ArticleItem(
|
||
source_id=source.id,
|
||
source_name=source.name,
|
||
url=url,
|
||
url_hash=url_hash,
|
||
title="",
|
||
crawl_time=now.isoformat(),
|
||
html_path="",
|
||
status="failed",
|
||
)
|
||
|
||
try:
|
||
config = _make_run_config(source)
|
||
result = await crawler.arun(url=url, config=config)
|
||
|
||
if not result.success:
|
||
article.error = result.error_message or "Unknown error"
|
||
logger.warning("[%s] [%d/%d] 抓取失败: %s",
|
||
source.id, index, total, article.error[:100])
|
||
return article
|
||
|
||
# 提取标题
|
||
article.title = (result.metadata.get("title") if result.metadata else "") or ""
|
||
|
||
# 保存输出
|
||
out_dir = Path(f"data/raw/{source.id}/{today}")
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# HTML
|
||
html_path = out_dir / f"{url_hash}.html"
|
||
if result.html:
|
||
html_path.write_text(result.html, encoding="utf-8")
|
||
article.html_path = str(html_path)
|
||
|
||
# Markdown
|
||
md_path = out_dir / f"{url_hash}.md"
|
||
md_content = ""
|
||
if hasattr(result, "markdown") and result.markdown:
|
||
md_content = str(result.markdown)
|
||
md_path.write_text(md_content, encoding="utf-8")
|
||
article.md_path = str(md_path)
|
||
|
||
article.word_count = len(md_content.split()) if md_content else 0
|
||
article.status = "success"
|
||
|
||
# 每篇都打日志方便追踪进度
|
||
mem_mb = _get_memory_mb()
|
||
logger.info("[%s] [%d/%d] ✅ %s (%d words, %.0f MB)",
|
||
source.id, index, total, article.title[:50], article.word_count, mem_mb)
|
||
|
||
except Exception as e:
|
||
article.error = str(e)
|
||
logger.error("[%s] [%d/%d] 抓取异常: %s", source.id, index, total, e)
|
||
|
||
return article
|
||
|
||
# ── 单源抓取 ──────────────────────────────────────────
|
||
|
||
|
||
async def crawl_source(source: SourceConfig) -> CrawlResult:
|
||
"""抓取单个新闻源：首页 → 串行抓取每篇文章
|
||
|
||
约束：内存 ≤ 2GB，超时 ≤ 2 小时
|
||
"""
|
||
start_time = datetime.now()
|
||
result = CrawlResult(
|
||
source_id=source.id,
|
||
source_name=source.name,
|
||
start_time=start_time.isoformat(),
|
||
)
|
||
|
||
mem_start = _get_memory_mb()
|
||
logger.info("━━━ 开始 [%s] %s (内存: %.0f MB) ━━━", source.id, source.name, mem_start)
|
||
|
||
browser_config = _make_browser_config(source)
|
||
|
||
try:
|
||
async with asyncio.timeout(SOURCE_TIMEOUT_SEC):
|
||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||
# 1. 首页提取链接
|
||
article_urls = await _extract_article_urls(crawler, source)
|
||
result.total_found = len(article_urls)
|
||
|
||
if not article_urls:
|
||
logger.warning("[%s] 未提取到任何文章链接", source.id)
|
||
result.end_time = datetime.now().isoformat()
|
||
return result
|
||
|
||
# 2. 串行抓取每篇文章
|
||
total = len(article_urls)
|
||
for i, url in enumerate(article_urls, 1):
|
||
_check_memory(source.id)
|
||
|
||
article = await _crawl_single_article(
|
||
crawler, url, source, index=i, total=total,
|
||
)
|
||
|
||
if article.status == "success":
|
||
result.total_success += 1
|
||
else:
|
||
result.total_failed += 1
|
||
result.articles.append(article)
|
||
|
||
# 冷却间隔
|
||
if i < total:
|
||
await asyncio.sleep(ARTICLE_DELAY_SEC)
|
||
|
||
except TimeoutError:
|
||
logger.error("[%s] ⏰ 超时 %.0f 秒（上限 %d 秒），中止",
|
||
source.id, (datetime.now() - start_time).total_seconds(),
|
||
SOURCE_TIMEOUT_SEC)
|
||
result.error = "Source timeout"
|
||
except Exception as e:
|
||
logger.exception("[%s] 抓取过程异常: %s", source.id, e)
|
||
result.error = str(e)
|
||
|
||
# 主动回收
|
||
gc.collect()
|
||
|
||
result.end_time = datetime.now().isoformat()
|
||
elapsed = (datetime.now() - start_time).total_seconds()
|
||
mem_end = _get_memory_mb()
|
||
logger.info(
|
||
"[%s] 完成: 发现 %d / 成功 %d / 失败 %d，耗时 %.1f 秒，内存 %.0f→%.0f MB",
|
||
source.id, result.total_found, result.total_success, result.total_failed,
|
||
elapsed, mem_start, mem_end,
|
||
)
|
||
|
||
return result
|