初始化
This commit is contained in:
@@ -0,0 +1,434 @@
|
||||
"""Crawl4AI 异步新闻抓取引擎
|
||||
|
||||
海外服务器运行，约束：内存 ≤ 2GB，串行抓取，单源 ≤ 2 小时。
|
||||
所有业务参数从 configs/system.yaml 的 crawler 节读取。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import psutil
|
||||
import yaml
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
|
||||
|
||||
from crawler.models import ArticleItem, CrawlResult, SourceConfig
|
||||
from crawler.utils import get_news_day
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 从配置文件加载常量 ──────────────────────────────────
|
||||
|
||||
|
||||
def _load_crawler_config() -> dict:
|
||||
"""从 system.yaml 读取 crawler 配置节（含 Profile 覆盖）"""
|
||||
try:
|
||||
from crawler.config import load_system_config
|
||||
cfg = load_system_config()
|
||||
return cfg.get("crawler", {})
|
||||
except Exception:
|
||||
logger.warning("读取 system.yaml 失败，使用默认值")
|
||||
return {}
|
||||
|
||||
_cfg = _load_crawler_config()
|
||||
|
||||
MAX_MEMORY_MB = int(_cfg.get("max_memory_mb", 1800))
|
||||
ARTICLE_DELAY_SEC = float(_cfg.get("article_delay_sec", 3.0))
|
||||
PAGE_TIMEOUT_MS = int(_cfg.get("page_timeout_sec", 45)) * 1000
|
||||
SOURCE_TIMEOUT_SEC = int(_cfg.get("source_timeout_sec", 7200))
|
||||
_VIEWPORT_WIDTH = int(_cfg.get("viewport_width", 1024))
|
||||
_VIEWPORT_HEIGHT = int(_cfg.get("viewport_height", 768))
|
||||
_USER_AGENT = _cfg.get(
|
||||
"user_agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/131.0.0.0 Safari/537.36",
|
||||
)
|
||||
_HEADFUL = bool(_cfg.get("headful", False))
|
||||
_XVFB_DISPLAY = _cfg.get("xvfb_display", ":99")
|
||||
|
||||
# ── 代理配置 ──────────────────────────────────────────
|
||||
|
||||
|
||||
def _load_proxy_config() -> dict:
|
||||
"""从 system.yaml 读取 proxy 配置节（含 Profile 覆盖）"""
|
||||
try:
|
||||
from crawler.config import load_system_config
|
||||
cfg = load_system_config()
|
||||
return cfg.get("proxy", {})
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
_proxy_cfg = _load_proxy_config()
|
||||
PROXY_ENABLED = bool(_proxy_cfg.get("enabled", False))
|
||||
PROXY_URL = _proxy_cfg.get("url", "socks5://127.0.0.1:1080")
|
||||
|
||||
# ── 工具函数 ──────────────────────────────────────────
|
||||
|
||||
|
||||
def compute_url_hash(url: str) -> str:
|
||||
"""计算 URL 的 SHA256 前 16 个字符作为短文件名"""
|
||||
return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _get_memory_mb() -> float:
|
||||
"""获取当前进程内存使用量（MB）"""
|
||||
proc = psutil.Process(os.getpid())
|
||||
return proc.memory_info().rss / (1024 * 1024)
|
||||
|
||||
|
||||
def _check_memory(source_id: str) -> None:
|
||||
"""检查内存，超限时警告"""
|
||||
mem = _get_memory_mb()
|
||||
if mem > MAX_MEMORY_MB:
|
||||
logger.warning("[%s] ⚠️ 内存使用 %.0f MB 超过上限 %.0f MB，触发 GC",
|
||||
source_id, mem, MAX_MEMORY_MB)
|
||||
gc.collect()
|
||||
mem_after = _get_memory_mb()
|
||||
logger.info("[%s] GC 后内存: %.0f MB (回收 %.0f MB)",
|
||||
source_id, mem_after, mem - mem_after)
|
||||
|
||||
# ── 浏览器配置 ──────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_browser_config(source: SourceConfig) -> BrowserConfig:
|
||||
"""浏览器配置：根据 anti_bot_mode 和系统配置选择策略
|
||||
|
||||
- None: 标准轻量 headless（默认）
|
||||
- "stealth": 反检测 headless（隐藏自动化特征）
|
||||
- "headful": 非 headless 模式（有头浏览器，最像真人）
|
||||
|
||||
代理：如果 system.yaml proxy.enabled=true，浏览器流量走 SOCKS5 代理。
|
||||
headful 模式自动设置 DISPLAY 环境变量（支持 xvfb）。
|
||||
"""
|
||||
mode = source.anti_bot_mode
|
||||
# headless 判定：
|
||||
# - 系统级 _HEADFUL=true → 强制有头（覆盖所有 source 模式）
|
||||
# - source mode="headful" → 有头
|
||||
# - 其他 → headless
|
||||
if _HEADFUL:
|
||||
headless = False # 系统级强制 headful
|
||||
elif mode == "headful":
|
||||
headless = False
|
||||
else:
|
||||
headless = True
|
||||
|
||||
# 基础反检测参数
|
||||
extra_args = [
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
"--no-sandbox",
|
||||
]
|
||||
|
||||
# 反检测参数：stealth 模式或系统级 headful 都启用
|
||||
if mode == "stealth" or _HEADFUL:
|
||||
extra_args += [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-features=IsolateOrigins,site-per-process",
|
||||
]
|
||||
|
||||
# ── SOCKS5 代理 ──────────────────────────────────
|
||||
if PROXY_ENABLED and PROXY_URL:
|
||||
extra_args.append(f"--proxy-server={PROXY_URL}")
|
||||
logger.info("🌐 浏览器代理已启用: %s", PROXY_URL)
|
||||
|
||||
# ── headful 模式：设置虚拟显示器 ──────────────────
|
||||
if not headless:
|
||||
display = os.environ.get("DISPLAY", "")
|
||||
if not display:
|
||||
os.environ["DISPLAY"] = _XVFB_DISPLAY
|
||||
logger.info("🖥️ headful 模式: DISPLAY=%s", _XVFB_DISPLAY)
|
||||
|
||||
return BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=headless,
|
||||
viewport_width=_VIEWPORT_WIDTH,
|
||||
viewport_height=_VIEWPORT_HEIGHT,
|
||||
verbose=False,
|
||||
text_mode=headless, # headful 模式下不禁用图片（更像真人）
|
||||
light_mode=headless,
|
||||
user_agent=_USER_AGENT,
|
||||
extra_args=extra_args,
|
||||
)
|
||||
|
||||
|
||||
def _make_run_config(source: SourceConfig) -> CrawlerRunConfig:
|
||||
"""抓取运行时配置"""
|
||||
return CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
page_timeout=PAGE_TIMEOUT_MS,
|
||||
wait_until="domcontentloaded",
|
||||
scan_full_page=False,
|
||||
simulate_user=True,
|
||||
override_navigator=True,
|
||||
)
|
||||
|
||||
def _extract_domain(url: str) -> str:
|
||||
"""从 URL 提取域名（去掉 www 前缀）。"""
|
||||
from urllib.parse import urlparse
|
||||
host = urlparse(url).hostname or ""
|
||||
return host.removeprefix("www.").lower()
|
||||
|
||||
|
||||
# ── 链接提取 ──────────────────────────────────────────
|
||||
|
||||
|
||||
async def _extract_article_urls(
|
||||
crawler: AsyncWebCrawler,
|
||||
source: SourceConfig,
|
||||
) -> list[str]:
|
||||
"""从首页抓取符合 article_url_pattern 的文章链接（增量：跳过已抓取过的 URL）"""
|
||||
config = _make_run_config(source)
|
||||
|
||||
try:
|
||||
result = await crawler.arun(url=source.homepage, config=config)
|
||||
except Exception as e:
|
||||
logger.error("抓取首页失败 [%s] %s: %s", source.id, source.homepage, e)
|
||||
return []
|
||||
|
||||
if not result.success:
|
||||
logger.error("首页返回失败 [%s]: %s", source.id, result.error_message)
|
||||
return []
|
||||
|
||||
html = result.html or ""
|
||||
pattern = re.compile(r'href=["\']([^"\']*?)["\']', re.IGNORECASE)
|
||||
raw_urls: list[str] = pattern.findall(html)
|
||||
|
||||
article_url_pattern = re.compile(source.article_url_pattern, re.IGNORECASE)
|
||||
base_without_fragment = source.homepage.split("#")[0]
|
||||
base_domain = _extract_domain(source.homepage)
|
||||
|
||||
# 非内容类扩展名
|
||||
_SKIP_EXT = re.compile(
|
||||
r"\.(png|ico|gif|jpg|jpeg|svg|webp|css|js|xml|json|rss|pdf|zip|woff2?|ttf|eot)"
|
||||
r"([?#]|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
# 非文章技术路径
|
||||
_SKIP_PATH = re.compile(
|
||||
r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)"
|
||||
r"[/.]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
article_urls: list[str] = []
|
||||
for raw in raw_urls:
|
||||
raw_stripped = raw.strip()
|
||||
if raw_stripped.startswith("#") or not raw_stripped:
|
||||
continue
|
||||
if raw_stripped.lower().startswith("javascript:"):
|
||||
continue
|
||||
|
||||
full = urljoin(source.homepage, raw_stripped)
|
||||
full_clean = full.split("#")[0]
|
||||
|
||||
# 过滤非内容类 URL
|
||||
if _SKIP_EXT.search(full_clean):
|
||||
continue
|
||||
if _SKIP_PATH.search(full_clean):
|
||||
continue
|
||||
# 过滤非同域链接（广告、CDN 等）
|
||||
if _extract_domain(full_clean) != base_domain:
|
||||
continue
|
||||
|
||||
if full_clean.rstrip("/") == base_without_fragment.rstrip("/"):
|
||||
continue
|
||||
if article_url_pattern.search(full_clean):
|
||||
article_urls.append(full_clean)
|
||||
|
||||
# 本批去重
|
||||
seen: set[str] = set()
|
||||
unique: list[str] = []
|
||||
for u in article_urls:
|
||||
if u not in seen:
|
||||
seen.add(u)
|
||||
unique.append(u)
|
||||
|
||||
# 增量过滤：读取已抓取的 url_hash，跳过已存在 URL
|
||||
today = get_news_day()
|
||||
index_path = Path(f"data/raw/{source.id}/{today}/index.jsonl")
|
||||
existing_hashes: set[str] = set()
|
||||
if index_path.exists():
|
||||
with open(index_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line)
|
||||
existing_hashes.add(item.get("url_hash", ""))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
filtered: list[str] = []
|
||||
skipped = 0
|
||||
for u in unique:
|
||||
uh = compute_url_hash(u)
|
||||
if uh in existing_hashes:
|
||||
skipped += 1
|
||||
logger.debug("[%s] 跳过已抓取: %s", source.id, u[:80])
|
||||
continue
|
||||
filtered.append(u)
|
||||
if len(filtered) >= source.max_articles_per_run:
|
||||
break
|
||||
|
||||
if skipped > 0:
|
||||
logger.info("[%s] 增量过滤: 跳过 %d 篇已抓取，剩余 %d 篇待抓取",
|
||||
source.id, skipped, len(filtered))
|
||||
|
||||
logger.info("[%s] 从首页提取到 %d 个文章链接（去重后 %d，增量后 %d）",
|
||||
source.id, len(article_urls), len(unique), len(filtered))
|
||||
return filtered
|
||||
|
||||
# ── 单篇抓取 ──────────────────────────────────────────
|
||||
|
||||
|
||||
async def _crawl_single_article(
|
||||
crawler: AsyncWebCrawler,
|
||||
url: str,
|
||||
source: SourceConfig,
|
||||
index: int,
|
||||
total: int,
|
||||
) -> ArticleItem:
|
||||
"""抓取单篇文章（串行模式）"""
|
||||
url_hash = compute_url_hash(url)
|
||||
now = datetime.now()
|
||||
today = get_news_day()
|
||||
|
||||
article = ArticleItem(
|
||||
source_id=source.id,
|
||||
source_name=source.name,
|
||||
url=url,
|
||||
url_hash=url_hash,
|
||||
title="",
|
||||
crawl_time=now.isoformat(),
|
||||
html_path="",
|
||||
status="failed",
|
||||
)
|
||||
|
||||
try:
|
||||
config = _make_run_config(source)
|
||||
result = await crawler.arun(url=url, config=config)
|
||||
|
||||
if not result.success:
|
||||
article.error = result.error_message or "Unknown error"
|
||||
logger.warning("[%s] [%d/%d] 抓取失败: %s",
|
||||
source.id, index, total, article.error[:100])
|
||||
return article
|
||||
|
||||
# 提取标题
|
||||
article.title = (result.metadata.get("title") if result.metadata else "") or ""
|
||||
|
||||
# 保存输出
|
||||
out_dir = Path(f"data/raw/{source.id}/{today}")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# HTML
|
||||
html_path = out_dir / f"{url_hash}.html"
|
||||
if result.html:
|
||||
html_path.write_text(result.html, encoding="utf-8")
|
||||
article.html_path = str(html_path)
|
||||
|
||||
# Markdown
|
||||
md_path = out_dir / f"{url_hash}.md"
|
||||
md_content = ""
|
||||
if hasattr(result, "markdown") and result.markdown:
|
||||
md_content = str(result.markdown)
|
||||
md_path.write_text(md_content, encoding="utf-8")
|
||||
article.md_path = str(md_path)
|
||||
|
||||
article.word_count = len(md_content.split()) if md_content else 0
|
||||
article.status = "success"
|
||||
|
||||
# 每篇都打日志方便追踪进度
|
||||
mem_mb = _get_memory_mb()
|
||||
logger.info("[%s] [%d/%d] ✅ %s (%d words, %.0f MB)",
|
||||
source.id, index, total, article.title[:50], article.word_count, mem_mb)
|
||||
|
||||
except Exception as e:
|
||||
article.error = str(e)
|
||||
logger.error("[%s] [%d/%d] 抓取异常: %s", source.id, index, total, e)
|
||||
|
||||
return article
|
||||
|
||||
# ── 单源抓取 ──────────────────────────────────────────
|
||||
|
||||
|
||||
async def crawl_source(source: SourceConfig) -> CrawlResult:
|
||||
"""抓取单个新闻源：首页 → 串行抓取每篇文章
|
||||
|
||||
约束：内存 ≤ 2GB，超时 ≤ 2 小时
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
result = CrawlResult(
|
||||
source_id=source.id,
|
||||
source_name=source.name,
|
||||
start_time=start_time.isoformat(),
|
||||
)
|
||||
|
||||
mem_start = _get_memory_mb()
|
||||
logger.info("━━━ 开始 [%s] %s (内存: %.0f MB) ━━━", source.id, source.name, mem_start)
|
||||
|
||||
browser_config = _make_browser_config(source)
|
||||
|
||||
try:
|
||||
async with asyncio.timeout(SOURCE_TIMEOUT_SEC):
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
# 1. 首页提取链接
|
||||
article_urls = await _extract_article_urls(crawler, source)
|
||||
result.total_found = len(article_urls)
|
||||
|
||||
if not article_urls:
|
||||
logger.warning("[%s] 未提取到任何文章链接", source.id)
|
||||
result.end_time = datetime.now().isoformat()
|
||||
return result
|
||||
|
||||
# 2. 串行抓取每篇文章
|
||||
total = len(article_urls)
|
||||
for i, url in enumerate(article_urls, 1):
|
||||
_check_memory(source.id)
|
||||
|
||||
article = await _crawl_single_article(
|
||||
crawler, url, source, index=i, total=total,
|
||||
)
|
||||
|
||||
if article.status == "success":
|
||||
result.total_success += 1
|
||||
else:
|
||||
result.total_failed += 1
|
||||
result.articles.append(article)
|
||||
|
||||
# 冷却间隔
|
||||
if i < total:
|
||||
await asyncio.sleep(ARTICLE_DELAY_SEC)
|
||||
|
||||
except TimeoutError:
|
||||
logger.error("[%s] ⏰ 超时 %.0f 秒（上限 %d 秒），中止",
|
||||
source.id, (datetime.now() - start_time).total_seconds(),
|
||||
SOURCE_TIMEOUT_SEC)
|
||||
result.error = "Source timeout"
|
||||
except Exception as e:
|
||||
logger.exception("[%s] 抓取过程异常: %s", source.id, e)
|
||||
result.error = str(e)
|
||||
|
||||
# 主动回收
|
||||
gc.collect()
|
||||
|
||||
result.end_time = datetime.now().isoformat()
|
||||
elapsed = (datetime.now() - start_time).total_seconds()
|
||||
mem_end = _get_memory_mb()
|
||||
logger.info(
|
||||
"[%s] 完成: 发现 %d / 成功 %d / 失败 %d，耗时 %.1f 秒，内存 %.0f→%.0f MB",
|
||||
source.id, result.total_found, result.total_success, result.total_failed,
|
||||
elapsed, mem_start, mem_end,
|
||||
)
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user