127 lines
4.5 KiB
Python
127 lines
4.5 KiB
Python
"""抓取编排器：串行调度多个新闻源的抓取。支持 RSS 优先、stealth/headful 回退。"""
|
||
|
||
import asyncio
|
||
import logging
|
||
from datetime import datetime
|
||
|
||
from crawler.crawler import crawl_source
|
||
from crawler.loader import load_sources
|
||
from crawler.models import CrawlResult, PipelineStats, SourceConfig
|
||
from crawler.rss_crawler import crawl_rss_source
|
||
from crawler.storage import write_index_jsonl
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
async def _crawl_single_source_with_storage(source: SourceConfig) -> CrawlResult:
|
||
"""抓取单个源并写入存储。
|
||
|
||
优先级:
|
||
1. 有 rss_url → RSS 抓取（绕过反爬）
|
||
2. RSS 失败或未配置 → Web 抓取（含 stealth/headful）
|
||
"""
|
||
result = await crawl_source_smart(source)
|
||
if result.total_success > 0:
|
||
write_index_jsonl(result)
|
||
return result
|
||
|
||
|
||
async def crawl_source_smart(source: SourceConfig) -> CrawlResult:
|
||
"""智能抓取：RSS 优先，Web 回退。
|
||
|
||
RSS 结果判定：
|
||
- total_success > 0 → 有新文章，直接返回
|
||
- total_found > 0, success=0 → 增量全部跳过（正常，不浪费 Web 回退）
|
||
- total_found == 0 → RSS 真·空结果
|
||
- error 非空 → RSS 失败（网络/解析错），回退 Web
|
||
|
||
Web 回退仅在 RSS 失败或未配置时执行。
|
||
"""
|
||
rss_url = getattr(source, "rss_url", None)
|
||
|
||
# ── 策略 1: RSS（同步，不消耗浏览器资源）──
|
||
if rss_url:
|
||
logger.info("[%s] 尝试 RSS 抓取: %s", source.id, rss_url)
|
||
rss_result = crawl_rss_source(source)
|
||
|
||
if rss_result.total_success > 0:
|
||
logger.info("[%s] ✅ RSS 成功: %d 篇", source.id, rss_result.total_success)
|
||
return rss_result
|
||
|
||
if rss_result.error:
|
||
# RSS 抓取本身失败（网络错/解析错）→ 回退 Web
|
||
logger.warning("[%s] RSS 失败 (%s)，回退 Web 抓取 (mode=%s)",
|
||
source.id, rss_result.error, source.anti_bot_mode)
|
||
elif rss_result.total_found > 0:
|
||
# RSS 成功但全部增量跳过 → 正常，不浪费 Web 资源
|
||
logger.info("[%s] RSS 无新文章（%d 条全部已抓取），跳过 Web 回退",
|
||
source.id, rss_result.total_found)
|
||
return rss_result
|
||
else:
|
||
# total_found == 0，RSS 返回空
|
||
logger.warning("[%s] RSS 返回空，回退 Web 抓取 (mode=%s)",
|
||
source.id, source.anti_bot_mode)
|
||
|
||
# ── 策略 2/3: Web 抓取（stealth / headful）──
|
||
mode = source.anti_bot_mode or "standard"
|
||
logger.info("[%s] 开始 Web 抓取 (mode=%s)", source.id, mode)
|
||
web_result = await crawl_source(source)
|
||
|
||
return web_result
|
||
|
||
|
||
async def crawl_all_sources(
|
||
source_filter: str | None = None,
|
||
) -> PipelineStats:
|
||
"""串行抓取所有启用的新闻源（海外服务器内存约束，逐个执行）
|
||
|
||
Args:
|
||
source_filter: 可选，只抓取指定 source_id
|
||
|
||
Returns:
|
||
PipelineStats 总体统计
|
||
"""
|
||
sources, _settings = load_sources()
|
||
|
||
if source_filter:
|
||
sources = [s for s in sources if s.id == source_filter]
|
||
if not sources:
|
||
raise ValueError(f"未找到启用的源: {source_filter}")
|
||
|
||
logger.info("══════ 开始串行抓取 %d 个新闻源 ══════", len(sources))
|
||
|
||
start_time = datetime.now()
|
||
stats = PipelineStats(start_time=start_time.isoformat(), sources_crawled=0)
|
||
|
||
# 串行执行每个源
|
||
for source in sources:
|
||
result = await _crawl_single_source_with_storage(source)
|
||
|
||
if isinstance(result, Exception):
|
||
logger.error("源抓取异常: %s", result)
|
||
stats.sources_failed += 1
|
||
else:
|
||
stats.sources_crawled += 1
|
||
stats.total_articles += result.total_success
|
||
stats.results.append(result)
|
||
if result.error:
|
||
stats.sources_failed += 1
|
||
|
||
stats.end_time = datetime.now().isoformat()
|
||
elapsed = (datetime.now() - start_time).total_seconds()
|
||
|
||
logger.info(
|
||
"══════ 抓取完成: %d/%d 源成功，共 %d 篇文章，耗时 %.1f 秒 ══════",
|
||
stats.sources_crawled - stats.sources_failed,
|
||
stats.sources_crawled,
|
||
stats.total_articles,
|
||
elapsed,
|
||
)
|
||
|
||
return stats
|
||
|
||
|
||
def run_crawl_sync(source_filter: str | None = None) -> PipelineStats:
|
||
"""同步包装器，供 CLI 调用"""
|
||
return asyncio.run(crawl_all_sources(source_filter=source_filter))
|