Initial commit
This commit is contained in:
@@ -0,0 +1,445 @@
|
||||
"""Crawl4AI 异步抓取引擎。
|
||||
|
||||
设计:
|
||||
1. 一个 AsyncWebCrawler 实例服务所有源(共用浏览器,降低开销);
|
||||
2. 每个 URL 独立的 CrawlerRunConfig(wait_for/timeout 来自源配置);
|
||||
3. 全局 Semaphore 控制并发(默认 3);
|
||||
4. 每个 URL 用 tenacity 重试(默认 3 次,指数退避);
|
||||
5. 两阶段: 先抓首页 -> 解析文章链接 -> 并发抓文章详情。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
|
||||
from loguru import logger
|
||||
from markdownify import markdownify as _md_convert
|
||||
|
||||
from .models import (
|
||||
ArticleLink,
|
||||
CrawlerConfig,
|
||||
CrawlerSettings,
|
||||
CrawlResult,
|
||||
CrawlStage,
|
||||
SourceConfig,
|
||||
)
|
||||
from .storage import mark_url_seen, save_result
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 工具函数
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _markdown_text(crawl4ai_markdown: Any) -> str:
|
||||
"""从 Crawl4AI 的 markdown 字段中安全取出字符串。
|
||||
|
||||
新版 Crawl4AI(0.4+) 返回 MarkdownGenerationResult 对象;
|
||||
旧版可能直接是 str。这里做兼容。
|
||||
"""
|
||||
if crawl4ai_markdown is None:
|
||||
return ""
|
||||
if isinstance(crawl4ai_markdown, str):
|
||||
return crawl4ai_markdown
|
||||
raw = getattr(crawl4ai_markdown, "raw_markdown", None)
|
||||
if isinstance(raw, str):
|
||||
return raw
|
||||
fit = getattr(crawl4ai_markdown, "fit_markdown", None)
|
||||
if isinstance(fit, str):
|
||||
return fit
|
||||
return str(crawl4ai_markdown)
|
||||
|
||||
|
||||
def extract_article_links(
|
||||
html: str,
|
||||
base_url: str,
|
||||
source: SourceConfig,
|
||||
) -> list[ArticleLink]:
|
||||
"""从列表页 HTML 中抽取文章链接,使用源配置的正则筛选。
|
||||
|
||||
步骤:
|
||||
1. 若设置了 article_link_selector,先 narrow 到选择器范围;
|
||||
2. 找出所有 <a href>;
|
||||
3. urljoin 转绝对 URL;
|
||||
4. 用 article_url_pattern 正则筛选;
|
||||
5. 去重保持顺序;
|
||||
6. 截断到 max_articles_per_run。
|
||||
"""
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
scope = (
|
||||
soup.select_one(source.article_link_selector)
|
||||
if source.article_link_selector
|
||||
else soup
|
||||
)
|
||||
if scope is None:
|
||||
logger.warning(
|
||||
"源 {} 设置了 article_link_selector={!r},但页面中未匹配到", source.id, source.article_link_selector
|
||||
)
|
||||
return []
|
||||
|
||||
pattern = re.compile(source.article_url_pattern)
|
||||
seen: set[str] = set()
|
||||
links: list[ArticleLink] = []
|
||||
|
||||
for a in scope.find_all("a", href=True):
|
||||
href = a["href"].strip()
|
||||
if not href or href.startswith(("javascript:", "#", "mailto:")):
|
||||
continue
|
||||
absolute = urljoin(base_url, href)
|
||||
# 去掉 fragment
|
||||
absolute = absolute.split("#", 1)[0]
|
||||
if not pattern.match(absolute):
|
||||
continue
|
||||
if absolute in seen:
|
||||
continue
|
||||
seen.add(absolute)
|
||||
anchor = (a.get_text() or "").strip() or None
|
||||
links.append(ArticleLink(source_id=source.id, url=absolute, anchor_text=anchor))
|
||||
if len(links) >= source.max_articles_per_run:
|
||||
break
|
||||
|
||||
logger.info(
|
||||
"源 {} 列表页抽取链接 {} 条 (上限 {})",
|
||||
source.id,
|
||||
len(links),
|
||||
source.max_articles_per_run,
|
||||
)
|
||||
return links
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 单 URL 抓取
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _build_run_config(source: SourceConfig) -> CrawlerRunConfig:
|
||||
"""根据源配置构造 CrawlerRunConfig。"""
|
||||
return CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
wait_for=source.wait_for,
|
||||
page_timeout=source.page_timeout_ms,
|
||||
excluded_tags=["script", "style", "noscript"],
|
||||
)
|
||||
|
||||
|
||||
class _TimeoutGuard:
|
||||
"""单源连续超时计数器，达到阈值后标记跳过，防止整个进程被拖死。"""
|
||||
|
||||
def __init__(self, source_id: str, max_consecutive: int = 2) -> None:
|
||||
self.source_id = source_id
|
||||
self.max_consecutive = max_consecutive
|
||||
self._count = 0
|
||||
self.skip = False
|
||||
|
||||
def record(self, error: str | None, success: bool) -> None:
|
||||
"""根据抓取结果更新计数器。"""
|
||||
if self.skip:
|
||||
return
|
||||
if success:
|
||||
self._count = 0
|
||||
return
|
||||
if error and "timeout" in error.lower():
|
||||
self._count += 1
|
||||
if self._count >= self.max_consecutive:
|
||||
self.skip = True
|
||||
logger.warning(
|
||||
"源 {} 连续超时 {} 次，跳过剩余请求",
|
||||
self.source_id,
|
||||
self._count,
|
||||
)
|
||||
|
||||
def skipped_result(self, url: str, stage: CrawlStage) -> CrawlResult:
|
||||
"""生成"已跳过"结果。"""
|
||||
return CrawlResult(
|
||||
source_id=self.source_id,
|
||||
stage=stage,
|
||||
url=url,
|
||||
success=False,
|
||||
error="Skipped: 源连续超时已跳过",
|
||||
attempts=0,
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_static(
|
||||
url: str,
|
||||
source: SourceConfig,
|
||||
stage: CrawlStage,
|
||||
attempt: int,
|
||||
) -> CrawlResult:
|
||||
"""使用 httpx 直连抓取静态页面（js_render=False），绕过 Playwright 反爬检测。"""
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
timeout_sec = source.page_timeout_ms / 1000.0
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout_sec,
|
||||
follow_redirects=True,
|
||||
headers=headers,
|
||||
) as client:
|
||||
r = await client.get(url)
|
||||
html = r.text
|
||||
# HTML → Markdown：先清洗再转换
|
||||
markdown = ""
|
||||
if html:
|
||||
try:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for tag in soup(["script", "style", "noscript", "meta", "link"]):
|
||||
tag.decompose()
|
||||
markdown = _md_convert(str(soup)) or ""
|
||||
except Exception:
|
||||
markdown = ""
|
||||
return CrawlResult(
|
||||
source_id=source.id,
|
||||
stage=stage,
|
||||
url=url,
|
||||
success=True,
|
||||
status_code=r.status_code,
|
||||
html=html,
|
||||
markdown=markdown,
|
||||
attempts=attempt,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("静态抓取异常 {} {}: {}", source.id, url, e)
|
||||
return CrawlResult(
|
||||
source_id=source.id,
|
||||
stage=stage,
|
||||
url=url,
|
||||
success=False,
|
||||
error=f"{type(e).__name__}: {e}",
|
||||
attempts=attempt,
|
||||
)
|
||||
|
||||
|
||||
async def _crawl_once(
|
||||
crawler: AsyncWebCrawler,
|
||||
url: str,
|
||||
source: SourceConfig,
|
||||
stage: CrawlStage,
|
||||
attempt: int,
|
||||
) -> CrawlResult:
|
||||
"""单次抓取(无重试),用于被 retry 循环包裹。
|
||||
|
||||
- js_render=False 且无 wait_for → httpx 直连,绕过浏览器反爬;
|
||||
- 其他情况 → Playwright。
|
||||
"""
|
||||
# 静态页面走 httpx 直连
|
||||
if not source.js_render and not source.wait_for:
|
||||
return await _fetch_static(url, source, stage, attempt)
|
||||
|
||||
# 动态页面走 Playwright
|
||||
run_config = _build_run_config(source)
|
||||
try:
|
||||
c4_result = await crawler.arun(url=url, config=run_config)
|
||||
except Exception as e:
|
||||
logger.warning("抓取异常 {} {}: {}", source.id, url, e)
|
||||
return CrawlResult(
|
||||
source_id=source.id,
|
||||
stage=stage,
|
||||
url=url,
|
||||
success=False,
|
||||
error=f"{type(e).__name__}: {e}",
|
||||
attempts=attempt,
|
||||
)
|
||||
|
||||
success = bool(getattr(c4_result, "success", False))
|
||||
return CrawlResult(
|
||||
source_id=source.id,
|
||||
stage=stage,
|
||||
url=url,
|
||||
success=success,
|
||||
status_code=getattr(c4_result, "status_code", None),
|
||||
html=getattr(c4_result, "html", "") or "",
|
||||
markdown=_markdown_text(getattr(c4_result, "markdown", None)),
|
||||
error=None if success else (getattr(c4_result, "error_message", None) or "unknown"),
|
||||
attempts=attempt,
|
||||
)
|
||||
|
||||
|
||||
async def crawl_url_with_retry(
|
||||
crawler: AsyncWebCrawler,
|
||||
url: str,
|
||||
source: SourceConfig,
|
||||
stage: CrawlStage,
|
||||
settings: CrawlerSettings,
|
||||
semaphore: asyncio.Semaphore,
|
||||
guard: _TimeoutGuard | None = None,
|
||||
) -> CrawlResult:
|
||||
"""带并发限制与指数退避重试的单 URL 抓取。
|
||||
|
||||
重试策略:
|
||||
最多 settings.retry_max_attempts 次;
|
||||
失败 (success=False) 触发重试,等待时间 min(min_wait * 2^(n-1), max_wait)。
|
||||
|
||||
若传入 guard,每次重试前检查是否已被跳过。
|
||||
"""
|
||||
async with semaphore:
|
||||
# 检查是否已被跳过
|
||||
if guard and guard.skip:
|
||||
return guard.skipped_result(url, stage)
|
||||
|
||||
max_attempts = settings.retry_max_attempts
|
||||
result: CrawlResult | None = None
|
||||
|
||||
for attempt_no in range(1, max_attempts + 1):
|
||||
result = await _crawl_once(crawler, url, source, stage, attempt_no)
|
||||
if result.success:
|
||||
if guard:
|
||||
guard.record(None, True)
|
||||
return result
|
||||
if attempt_no >= max_attempts:
|
||||
break
|
||||
# 重试前检查是否已被其他并发请求触发跳过
|
||||
if guard and guard.skip:
|
||||
return guard.skipped_result(url, stage)
|
||||
wait = min(
|
||||
settings.retry_min_wait_sec * (2 ** (attempt_no - 1)),
|
||||
settings.retry_max_wait_sec,
|
||||
)
|
||||
logger.info(
|
||||
"源 {} {} 第 {}/{} 次失败,{} 秒后重试: {}",
|
||||
source.id,
|
||||
url,
|
||||
attempt_no,
|
||||
max_attempts,
|
||||
wait,
|
||||
result.error,
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
# 此处 result 必非 None(循环至少跑一次)
|
||||
assert result is not None
|
||||
if guard:
|
||||
guard.record(result.error, False)
|
||||
return result
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 抓取流程
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
async def crawl_source(
|
||||
crawler: AsyncWebCrawler,
|
||||
source: SourceConfig,
|
||||
settings: CrawlerSettings,
|
||||
semaphore: asyncio.Semaphore,
|
||||
save: bool = True,
|
||||
) -> list[CrawlResult]:
|
||||
"""抓取单个新闻源:遍历所有入口 → 抽取链接(去重) → 并发抓文章。"""
|
||||
homepages = source.all_homepages()
|
||||
logger.info(
|
||||
"===== 开始抓取源: {} ({}) {} 个入口 =====",
|
||||
source.id, source.name, len(homepages),
|
||||
)
|
||||
|
||||
all_list_results: list[CrawlResult] = []
|
||||
all_links: list[ArticleLink] = []
|
||||
|
||||
# 增量:加载历史 URL hash
|
||||
from .storage import load_seen_urls
|
||||
from .storage import url_hash as storage_url_hash
|
||||
seen_hashes = load_seen_urls(source.id)
|
||||
if seen_hashes:
|
||||
logger.info("源 {} 增量模式: 已记录 {} 条历史 URL", source.id, len(seen_hashes))
|
||||
|
||||
# 遍历每个入口
|
||||
for hp_url in homepages:
|
||||
list_result = await crawl_url_with_retry(
|
||||
crawler=crawler, url=hp_url, source=source,
|
||||
stage=CrawlStage.LIST, settings=settings, semaphore=semaphore,
|
||||
)
|
||||
if save:
|
||||
save_result(list_result, settings.output_root)
|
||||
all_list_results.append(list_result)
|
||||
|
||||
if not list_result.success:
|
||||
logger.warning("源 {} 入口 {} 抓取失败,跳过: {}", source.id, hp_url, list_result.error)
|
||||
continue
|
||||
|
||||
parsed_base = urlparse(hp_url)
|
||||
base = f"{parsed_base.scheme}://{parsed_base.netloc}"
|
||||
links = extract_article_links(list_result.html, base, source)
|
||||
|
||||
# 去重 + 增量:按 hash 去重(当前运行跨入口 + 历史记录)
|
||||
new_links = [ln for ln in links if storage_url_hash(ln.url) not in seen_hashes]
|
||||
for ln in new_links:
|
||||
seen_hashes.add(storage_url_hash(ln.url))
|
||||
all_links.extend(new_links)
|
||||
# 旧链接数(跨入口去重前)
|
||||
old_count = len(links) - len(new_links)
|
||||
logger.info("源 {} 入口 {} 抽取链接 {} 条(去重后新增 {}, 跳过旧 {} 条)",
|
||||
source.id, hp_url, len(links), len(new_links), old_count)
|
||||
|
||||
if not all_links:
|
||||
logger.warning("源 {} 所有入口未发现任何文章链接", source.id)
|
||||
return all_list_results
|
||||
|
||||
# 连续超时保护：同源连续超时 2 次后跳过剩余请求
|
||||
guard = _TimeoutGuard(source.id)
|
||||
|
||||
article_tasks = [
|
||||
crawl_url_with_retry(
|
||||
crawler=crawler, url=link.url, source=source,
|
||||
stage=CrawlStage.ARTICLE, settings=settings, semaphore=semaphore,
|
||||
guard=guard,
|
||||
)
|
||||
for link in all_links
|
||||
]
|
||||
article_results = await asyncio.gather(*article_tasks, return_exceptions=False)
|
||||
|
||||
if save:
|
||||
for r in article_results:
|
||||
save_result(r, settings.output_root)
|
||||
|
||||
success_cnt = sum(1 for r in article_results if r.success)
|
||||
# 记录成功抓取的 URL(增量去重)
|
||||
new_saved = 0
|
||||
for r in article_results:
|
||||
if r.success:
|
||||
mark_url_seen(source.id, r.url)
|
||||
new_saved += 1
|
||||
|
||||
logger.info(
|
||||
"源 {} 完成: 文章 {}/{} 成功, 新增 {} 条已记录",
|
||||
source.id, success_cnt, len(article_results), new_saved,
|
||||
)
|
||||
return [*all_list_results, *article_results]
|
||||
|
||||
|
||||
async def crawl_all(
|
||||
config: CrawlerConfig,
|
||||
save: bool = True,
|
||||
) -> list[CrawlResult]:
|
||||
"""抓取所有启用的源,返回扁平的 CrawlResult 列表。"""
|
||||
settings = config.settings
|
||||
enabled = config.enabled_sources()
|
||||
if not enabled:
|
||||
logger.warning("没有启用的新闻源,直接返回")
|
||||
return []
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=settings.headless,
|
||||
user_agent=settings.user_agent,
|
||||
verbose=False,
|
||||
)
|
||||
semaphore = asyncio.Semaphore(settings.concurrency)
|
||||
all_results: list[CrawlResult] = []
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
for source in enabled:
|
||||
try:
|
||||
results = await crawl_source(crawler, source, settings, semaphore, save=save)
|
||||
all_results.extend(results)
|
||||
except Exception as e: # 单源异常不应中断整体抓取
|
||||
logger.exception("源 {} 抓取异常: {}", source.id, e)
|
||||
|
||||
total = len(all_results)
|
||||
succ = sum(1 for r in all_results if r.success)
|
||||
logger.info("全部完成: {}/{} 成功 (成功率 {:.0%})", succ, total, succ / max(total, 1))
|
||||
return all_results
|
||||
Reference in New Issue
Block a user