初始化

crawler/config.py

@@ -0,0 +1,59 @@
+"""系统配置加载 + Profile 覆盖。
+
+读取 configs/system.yaml，如果设置了 EN_NEWS_PROFILE 环境变量，
+则加载 configs/profiles/{name}.yaml 并深度合并覆盖。
+"""
+
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+SYSTEM_CONFIG_PATH = Path("configs/system.yaml")
+PROFILES_DIR = Path("configs/profiles")
+
+
+def _deep_merge(base: dict, override: dict) -> dict:
+    """深度合并两个字典，override 的值覆盖 base。"""
+    result = base.copy()
+    for key, value in override.items():
+        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
+            result[key] = _deep_merge(result[key], value)
+        else:
+            result[key] = value
+    return result
+
+
+def load_system_config() -> dict[str, Any]:
+    """加载系统配置，自动应用 EN_NEWS_PROFILE 覆盖。
+
+    Returns:
+        合并后的完整配置字典（已应用 profile 覆盖）
+    """
+    if not SYSTEM_CONFIG_PATH.exists():
+        raise FileNotFoundError(f"系统配置文件不存在: {SYSTEM_CONFIG_PATH}")
+
+    with open(SYSTEM_CONFIG_PATH, encoding="utf-8") as f:
+        config: dict[str, Any] = yaml.safe_load(f) or {}
+
+    # ── Profile 覆盖 ──────────────────────────────────
+    profile_name = os.environ.get("EN_NEWS_PROFILE", "")
+    if profile_name:
+        profile_path = PROFILES_DIR / f"{profile_name}.yaml"
+        if profile_path.exists():
+            logger.info("📋 加载 Profile: %s (%s)", profile_name, profile_path)
+            with open(profile_path, encoding="utf-8") as f:
+                profile_cfg = yaml.safe_load(f) or {}
+            config = _deep_merge(config, profile_cfg)
+            logger.info("📋 Profile 覆盖已应用: proxy=%s, headful=%s, max_memory=%s",
+                        config.get("proxy", {}).get("enabled"),
+                        config.get("crawler", {}).get("headful"),
+                        config.get("crawler", {}).get("max_memory_mb"))
+        else:
+            logger.warning("⚠️  Profile 文件不存在: %s", profile_path)
+
+    return config

crawler/crawler.py

@@ -0,0 +1,434 @@
+"""Crawl4AI 异步新闻抓取引擎
+
+海外服务器运行，约束：内存 ≤ 2GB，串行抓取，单源 ≤ 2 小时。
+所有业务参数从 configs/system.yaml 的 crawler 节读取。
+"""
+
+import asyncio
+import gc
+import hashlib
+import json
+import logging
+import os
+import re
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import urljoin
+
+import psutil
+import yaml
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
+
+from crawler.models import ArticleItem, CrawlResult, SourceConfig
+from crawler.utils import get_news_day
+
+logger = logging.getLogger(__name__)
+
+# ── 从配置文件加载常量 ──────────────────────────────────
+
+
+def _load_crawler_config() -> dict:
+    """从 system.yaml 读取 crawler 配置节（含 Profile 覆盖）"""
+    try:
+        from crawler.config import load_system_config
+        cfg = load_system_config()
+        return cfg.get("crawler", {})
+    except Exception:
+        logger.warning("读取 system.yaml 失败，使用默认值")
+    return {}
+
+_cfg = _load_crawler_config()
+
+MAX_MEMORY_MB = int(_cfg.get("max_memory_mb", 1800))
+ARTICLE_DELAY_SEC = float(_cfg.get("article_delay_sec", 3.0))
+PAGE_TIMEOUT_MS = int(_cfg.get("page_timeout_sec", 45)) * 1000
+SOURCE_TIMEOUT_SEC = int(_cfg.get("source_timeout_sec", 7200))
+_VIEWPORT_WIDTH = int(_cfg.get("viewport_width", 1024))
+_VIEWPORT_HEIGHT = int(_cfg.get("viewport_height", 768))
+_USER_AGENT = _cfg.get(
+    "user_agent",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/131.0.0.0 Safari/537.36",
+)
+_HEADFUL = bool(_cfg.get("headful", False))
+_XVFB_DISPLAY = _cfg.get("xvfb_display", ":99")
+
+# ── 代理配置 ──────────────────────────────────────────
+
+
+def _load_proxy_config() -> dict:
+    """从 system.yaml 读取 proxy 配置节（含 Profile 覆盖）"""
+    try:
+        from crawler.config import load_system_config
+        cfg = load_system_config()
+        return cfg.get("proxy", {})
+    except Exception:
+        pass
+    return {}
+
+_proxy_cfg = _load_proxy_config()
+PROXY_ENABLED = bool(_proxy_cfg.get("enabled", False))
+PROXY_URL = _proxy_cfg.get("url", "socks5://127.0.0.1:1080")
+
+# ── 工具函数 ──────────────────────────────────────────
+
+
+def compute_url_hash(url: str) -> str:
+    """计算 URL 的 SHA256 前 16 个字符作为短文件名"""
+    return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
+
+
+def _get_memory_mb() -> float:
+    """获取当前进程内存使用量（MB）"""
+    proc = psutil.Process(os.getpid())
+    return proc.memory_info().rss / (1024 * 1024)
+
+
+def _check_memory(source_id: str) -> None:
+    """检查内存，超限时警告"""
+    mem = _get_memory_mb()
+    if mem > MAX_MEMORY_MB:
+        logger.warning("[%s] ⚠️ 内存使用 %.0f MB 超过上限 %.0f MB，触发 GC",
+                       source_id, mem, MAX_MEMORY_MB)
+        gc.collect()
+        mem_after = _get_memory_mb()
+        logger.info("[%s] GC 后内存: %.0f MB (回收 %.0f MB)",
+                    source_id, mem_after, mem - mem_after)
+
+# ── 浏览器配置 ──────────────────────────────────────────
+
+
+def _make_browser_config(source: SourceConfig) -> BrowserConfig:
+    """浏览器配置：根据 anti_bot_mode 和系统配置选择策略
+
+    - None:      标准轻量 headless（默认）
+    - "stealth": 反检测 headless（隐藏自动化特征）
+    - "headful": 非 headless 模式（有头浏览器，最像真人）
+
+    代理：如果 system.yaml proxy.enabled=true，浏览器流量走 SOCKS5 代理。
+    headful 模式自动设置 DISPLAY 环境变量（支持 xvfb）。
+    """
+    mode = source.anti_bot_mode
+    # headless 判定：
+    #   - 系统级 _HEADFUL=true  → 强制有头（覆盖所有 source 模式）
+    #   - source mode="headful"  → 有头
+    #   - 其他                   → headless
+    if _HEADFUL:
+        headless = False        # 系统级强制 headful
+    elif mode == "headful":
+        headless = False
+    else:
+        headless = True
+
+    # 基础反检测参数
+    extra_args = [
+        "--disable-dev-shm-usage",
+        "--disable-gpu",
+        "--no-sandbox",
+    ]
+
+    # 反检测参数：stealth 模式或系统级 headful 都启用
+    if mode == "stealth" or _HEADFUL:
+        extra_args += [
+            "--disable-blink-features=AutomationControlled",
+            "--disable-features=IsolateOrigins,site-per-process",
+        ]
+
+    # ── SOCKS5 代理 ──────────────────────────────────
+    if PROXY_ENABLED and PROXY_URL:
+        extra_args.append(f"--proxy-server={PROXY_URL}")
+        logger.info("🌐 浏览器代理已启用: %s", PROXY_URL)
+
+    # ── headful 模式：设置虚拟显示器 ──────────────────
+    if not headless:
+        display = os.environ.get("DISPLAY", "")
+        if not display:
+            os.environ["DISPLAY"] = _XVFB_DISPLAY
+            logger.info("🖥️  headful 模式: DISPLAY=%s", _XVFB_DISPLAY)
+
+    return BrowserConfig(
+        browser_type="chromium",
+        headless=headless,
+        viewport_width=_VIEWPORT_WIDTH,
+        viewport_height=_VIEWPORT_HEIGHT,
+        verbose=False,
+        text_mode=headless,             # headful 模式下不禁用图片（更像真人）
+        light_mode=headless,
+        user_agent=_USER_AGENT,
+        extra_args=extra_args,
+    )
+
+
+def _make_run_config(source: SourceConfig) -> CrawlerRunConfig:
+    """抓取运行时配置"""
+    return CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        page_timeout=PAGE_TIMEOUT_MS,
+        wait_until="domcontentloaded",
+        scan_full_page=False,
+        simulate_user=True,
+        override_navigator=True,
+    )
+
+def _extract_domain(url: str) -> str:
+    """从 URL 提取域名（去掉 www 前缀）。"""
+    from urllib.parse import urlparse
+    host = urlparse(url).hostname or ""
+    return host.removeprefix("www.").lower()
+
+
+# ── 链接提取 ──────────────────────────────────────────
+
+
+async def _extract_article_urls(
+    crawler: AsyncWebCrawler,
+    source: SourceConfig,
+) -> list[str]:
+    """从首页抓取符合 article_url_pattern 的文章链接（增量：跳过已抓取过的 URL）"""
+    config = _make_run_config(source)
+
+    try:
+        result = await crawler.arun(url=source.homepage, config=config)
+    except Exception as e:
+        logger.error("抓取首页失败 [%s] %s: %s", source.id, source.homepage, e)
+        return []
+
+    if not result.success:
+        logger.error("首页返回失败 [%s]: %s", source.id, result.error_message)
+        return []
+
+    html = result.html or ""
+    pattern = re.compile(r'href=["\']([^"\']*?)["\']', re.IGNORECASE)
+    raw_urls: list[str] = pattern.findall(html)
+
+    article_url_pattern = re.compile(source.article_url_pattern, re.IGNORECASE)
+    base_without_fragment = source.homepage.split("#")[0]
+    base_domain = _extract_domain(source.homepage)
+
+    # 非内容类扩展名
+    _SKIP_EXT = re.compile(
+        r"\.(png|ico|gif|jpg|jpeg|svg|webp|css|js|xml|json|rss|pdf|zip|woff2?|ttf|eot)"
+        r"([?#]|$)",
+        re.IGNORECASE,
+    )
+    # 非文章技术路径
+    _SKIP_PATH = re.compile(
+        r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)"
+        r"[/.]",
+        re.IGNORECASE,
+    )
+
+    article_urls: list[str] = []
+    for raw in raw_urls:
+        raw_stripped = raw.strip()
+        if raw_stripped.startswith("#") or not raw_stripped:
+            continue
+        if raw_stripped.lower().startswith("javascript:"):
+            continue
+
+        full = urljoin(source.homepage, raw_stripped)
+        full_clean = full.split("#")[0]
+
+        # 过滤非内容类 URL
+        if _SKIP_EXT.search(full_clean):
+            continue
+        if _SKIP_PATH.search(full_clean):
+            continue
+        # 过滤非同域链接（广告、CDN 等）
+        if _extract_domain(full_clean) != base_domain:
+            continue
+
+        if full_clean.rstrip("/") == base_without_fragment.rstrip("/"):
+            continue
+        if article_url_pattern.search(full_clean):
+            article_urls.append(full_clean)
+
+    # 本批去重
+    seen: set[str] = set()
+    unique: list[str] = []
+    for u in article_urls:
+        if u not in seen:
+            seen.add(u)
+            unique.append(u)
+
+    # 增量过滤：读取已抓取的 url_hash，跳过已存在 URL
+    today = get_news_day()
+    index_path = Path(f"data/raw/{source.id}/{today}/index.jsonl")
+    existing_hashes: set[str] = set()
+    if index_path.exists():
+        with open(index_path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    item = json.loads(line)
+                    existing_hashes.add(item.get("url_hash", ""))
+                except json.JSONDecodeError:
+                    continue
+
+    filtered: list[str] = []
+    skipped = 0
+    for u in unique:
+        uh = compute_url_hash(u)
+        if uh in existing_hashes:
+            skipped += 1
+            logger.debug("[%s] 跳过已抓取: %s", source.id, u[:80])
+            continue
+        filtered.append(u)
+        if len(filtered) >= source.max_articles_per_run:
+            break
+
+    if skipped > 0:
+        logger.info("[%s] 增量过滤: 跳过 %d 篇已抓取，剩余 %d 篇待抓取",
+                    source.id, skipped, len(filtered))
+
+    logger.info("[%s] 从首页提取到 %d 个文章链接（去重后 %d，增量后 %d）",
+                source.id, len(article_urls), len(unique), len(filtered))
+    return filtered
+
+# ── 单篇抓取 ──────────────────────────────────────────
+
+
+async def _crawl_single_article(
+    crawler: AsyncWebCrawler,
+    url: str,
+    source: SourceConfig,
+    index: int,
+    total: int,
+) -> ArticleItem:
+    """抓取单篇文章（串行模式）"""
+    url_hash = compute_url_hash(url)
+    now = datetime.now()
+    today = get_news_day()
+
+    article = ArticleItem(
+        source_id=source.id,
+        source_name=source.name,
+        url=url,
+        url_hash=url_hash,
+        title="",
+        crawl_time=now.isoformat(),
+        html_path="",
+        status="failed",
+    )
+
+    try:
+        config = _make_run_config(source)
+        result = await crawler.arun(url=url, config=config)
+
+        if not result.success:
+            article.error = result.error_message or "Unknown error"
+            logger.warning("[%s] [%d/%d] 抓取失败: %s",
+                           source.id, index, total, article.error[:100])
+            return article
+
+        # 提取标题
+        article.title = (result.metadata.get("title") if result.metadata else "") or ""
+
+        # 保存输出
+        out_dir = Path(f"data/raw/{source.id}/{today}")
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+        # HTML
+        html_path = out_dir / f"{url_hash}.html"
+        if result.html:
+            html_path.write_text(result.html, encoding="utf-8")
+            article.html_path = str(html_path)
+
+        # Markdown
+        md_path = out_dir / f"{url_hash}.md"
+        md_content = ""
+        if hasattr(result, "markdown") and result.markdown:
+            md_content = str(result.markdown)
+            md_path.write_text(md_content, encoding="utf-8")
+            article.md_path = str(md_path)
+
+        article.word_count = len(md_content.split()) if md_content else 0
+        article.status = "success"
+
+        # 每篇都打日志方便追踪进度
+        mem_mb = _get_memory_mb()
+        logger.info("[%s] [%d/%d] ✅ %s (%d words, %.0f MB)",
+                    source.id, index, total, article.title[:50], article.word_count, mem_mb)
+
+    except Exception as e:
+        article.error = str(e)
+        logger.error("[%s] [%d/%d] 抓取异常: %s", source.id, index, total, e)
+
+    return article
+
+# ── 单源抓取 ──────────────────────────────────────────
+
+
+async def crawl_source(source: SourceConfig) -> CrawlResult:
+    """抓取单个新闻源：首页 → 串行抓取每篇文章
+
+    约束：内存 ≤ 2GB，超时 ≤ 2 小时
+    """
+    start_time = datetime.now()
+    result = CrawlResult(
+        source_id=source.id,
+        source_name=source.name,
+        start_time=start_time.isoformat(),
+    )
+
+    mem_start = _get_memory_mb()
+    logger.info("━━━ 开始 [%s] %s (内存: %.0f MB) ━━━", source.id, source.name, mem_start)
+
+    browser_config = _make_browser_config(source)
+
+    try:
+        async with asyncio.timeout(SOURCE_TIMEOUT_SEC):
+            async with AsyncWebCrawler(config=browser_config) as crawler:
+                # 1. 首页提取链接
+                article_urls = await _extract_article_urls(crawler, source)
+                result.total_found = len(article_urls)
+
+                if not article_urls:
+                    logger.warning("[%s] 未提取到任何文章链接", source.id)
+                    result.end_time = datetime.now().isoformat()
+                    return result
+
+                # 2. 串行抓取每篇文章
+                total = len(article_urls)
+                for i, url in enumerate(article_urls, 1):
+                    _check_memory(source.id)
+
+                    article = await _crawl_single_article(
+                        crawler, url, source, index=i, total=total,
+                    )
+
+                    if article.status == "success":
+                        result.total_success += 1
+                    else:
+                        result.total_failed += 1
+                    result.articles.append(article)
+
+                    # 冷却间隔
+                    if i < total:
+                        await asyncio.sleep(ARTICLE_DELAY_SEC)
+
+    except TimeoutError:
+        logger.error("[%s] ⏰ 超时 %.0f 秒（上限 %d 秒），中止",
+                     source.id, (datetime.now() - start_time).total_seconds(),
+                     SOURCE_TIMEOUT_SEC)
+        result.error = "Source timeout"
+    except Exception as e:
+        logger.exception("[%s] 抓取过程异常: %s", source.id, e)
+        result.error = str(e)
+
+    # 主动回收
+    gc.collect()
+
+    result.end_time = datetime.now().isoformat()
+    elapsed = (datetime.now() - start_time).total_seconds()
+    mem_end = _get_memory_mb()
+    logger.info(
+        "[%s] 完成: 发现 %d / 成功 %d / 失败 %d，耗时 %.1f 秒，内存 %.0f→%.0f MB",
+        source.id, result.total_found, result.total_success, result.total_failed,
+        elapsed, mem_start, mem_end,
+    )
+
+    return result

crawler/loader.py

@@ -0,0 +1,65 @@
+"""加载和管理新闻源配置"""
+
+import logging
+from pathlib import Path
+
+import yaml
+
+from crawler.models import SourceConfig
+
+logger = logging.getLogger(__name__)
+
+# 默认配置文件路径
+DEFAULT_SOURCES_PATH = Path("configs/sources.yaml")
+
+
+def load_sources(
+    config_path: Path | None = None,
+) -> tuple[list[SourceConfig], dict]:
+    """从 YAML 加载新闻源配置
+
+    Args:
+        config_path: 配置文件路径，默认 configs/sources.yaml
+
+    Returns:
+        (启用的源列表, settings dict)
+    """
+    path = config_path or DEFAULT_SOURCES_PATH
+
+    if not path.exists():
+        raise FileNotFoundError(f"新闻源配置文件不存在: {path}")
+
+    with open(path, encoding="utf-8") as f:
+        raw = yaml.safe_load(f)
+
+    if not raw or "sources" not in raw:
+        raise ValueError(f"配置文件格式错误，缺少 'sources' 字段: {path}")
+
+    settings = raw.get("settings", {})
+    sources: list[SourceConfig] = []
+
+    for item in raw["sources"]:
+        if not isinstance(item, dict):
+            logger.warning("跳过非字典格式的源配置: %s", item)
+            continue
+
+        if not item.get("enabled", True):
+            logger.info("跳过已禁用的源: %s (%s)", item.get("id"), item.get("name"))
+            continue
+
+        try:
+            source = SourceConfig(**item)
+            sources.append(source)
+        except Exception as e:
+            logger.error("解析源配置失败: %s - %s", item.get("id"), e)
+
+    logger.info("成功加载 %d 个启用的新闻源（共 %d 个）", len(sources), len(raw["sources"]))
+    return sources, settings
+
+
+def get_source_by_id(source_id: str, sources: list[SourceConfig]) -> SourceConfig | None:
+    """按 ID 查找源"""
+    for s in sources:
+        if s.id == source_id:
+            return s
+    return None

crawler/models.py

@@ -0,0 +1,68 @@
+"""爬虫数据模型"""
+
+from datetime import datetime
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+
+class SourceConfig(BaseModel):
+    """单个新闻源配置"""
+
+    id: str
+    name: str
+    enabled: bool = True
+    homepage: str
+    article_url_pattern: str
+    js_render: bool = False
+    max_articles_per_run: int = 30
+    rss_url: str | None = None           # RSS/Atom feed URL（优先使用，绕过反爬）
+    anti_bot_mode: str | None = None      # "stealth" | "headful" | None（反爬策略）
+
+    @property
+    def output_dir(self) -> Path:
+        """按日期组织的输出目录"""
+        today = datetime.now().strftime("%Y%m%d")
+        return Path(f"data/raw/{self.id}/{today}")
+
+
+class ArticleItem(BaseModel):
+    """单篇已抓取的文章元数据"""
+
+    source_id: str
+    source_name: str
+    url: str
+    url_hash: str
+    title: str
+    crawl_time: str  # ISO 8601
+    publish_time: str = ""  # ISO 8601，RSS 源可提取，网页源由 extractor 补充
+    html_path: str  # 相对路径，如 data/raw/reuters/20260621/abc123.html
+    md_path: str = ""  # Crawl4AI 生成的 Markdown 路径
+    word_count: int = 0
+    status: str = "success"  # success | failed
+    error: str = ""
+
+
+class CrawlResult(BaseModel):
+    """单次抓取结果统计"""
+
+    source_id: str
+    source_name: str
+    total_found: int = 0
+    total_success: int = 0
+    total_failed: int = 0
+    articles: list[ArticleItem] = Field(default_factory=list)
+    start_time: str = ""
+    end_time: str = ""
+    error: str = ""
+
+
+class PipelineStats(BaseModel):
+    """一次完整抓取管道的统计"""
+
+    start_time: str = ""
+    end_time: str = ""
+    sources_crawled: int = 0
+    sources_failed: int = 0
+    total_articles: int = 0
+    results: list[CrawlResult] = Field(default_factory=list)

crawler/orchestrator.py

@@ -0,0 +1,126 @@
+"""抓取编排器：串行调度多个新闻源的抓取。支持 RSS 优先、stealth/headful 回退。"""
+
+import asyncio
+import logging
+from datetime import datetime
+
+from crawler.crawler import crawl_source
+from crawler.loader import load_sources
+from crawler.models import CrawlResult, PipelineStats, SourceConfig
+from crawler.rss_crawler import crawl_rss_source
+from crawler.storage import write_index_jsonl
+
+logger = logging.getLogger(__name__)
+
+
+async def _crawl_single_source_with_storage(source: SourceConfig) -> CrawlResult:
+    """抓取单个源并写入存储。
+
+    优先级:
+        1. 有 rss_url → RSS 抓取（绕过反爬）
+        2. RSS 失败或未配置 → Web 抓取（含 stealth/headful）
+    """
+    result = await crawl_source_smart(source)
+    if result.total_success > 0:
+        write_index_jsonl(result)
+    return result
+
+
+async def crawl_source_smart(source: SourceConfig) -> CrawlResult:
+    """智能抓取：RSS 优先，Web 回退。
+
+    RSS 结果判定：
+    - total_success > 0          → 有新文章，直接返回
+    - total_found > 0, success=0 → 增量全部跳过（正常，不浪费 Web 回退）
+    - total_found == 0           → RSS 真·空结果
+    - error 非空                 → RSS 失败（网络/解析错），回退 Web
+
+    Web 回退仅在 RSS 失败或未配置时执行。
+    """
+    rss_url = getattr(source, "rss_url", None)
+
+    # ── 策略 1: RSS（同步，不消耗浏览器资源）──
+    if rss_url:
+        logger.info("[%s] 尝试 RSS 抓取: %s", source.id, rss_url)
+        rss_result = crawl_rss_source(source)
+
+        if rss_result.total_success > 0:
+            logger.info("[%s] ✅ RSS 成功: %d 篇", source.id, rss_result.total_success)
+            return rss_result
+
+        if rss_result.error:
+            # RSS 抓取本身失败（网络错/解析错）→ 回退 Web
+            logger.warning("[%s] RSS 失败 (%s)，回退 Web 抓取 (mode=%s)",
+                          source.id, rss_result.error, source.anti_bot_mode)
+        elif rss_result.total_found > 0:
+            # RSS 成功但全部增量跳过 → 正常，不浪费 Web 资源
+            logger.info("[%s] RSS 无新文章（%d 条全部已抓取），跳过 Web 回退",
+                       source.id, rss_result.total_found)
+            return rss_result
+        else:
+            # total_found == 0，RSS 返回空
+            logger.warning("[%s] RSS 返回空，回退 Web 抓取 (mode=%s)",
+                          source.id, source.anti_bot_mode)
+
+    # ── 策略 2/3: Web 抓取（stealth / headful）──
+    mode = source.anti_bot_mode or "standard"
+    logger.info("[%s] 开始 Web 抓取 (mode=%s)", source.id, mode)
+    web_result = await crawl_source(source)
+
+    return web_result
+
+
+async def crawl_all_sources(
+    source_filter: str | None = None,
+) -> PipelineStats:
+    """串行抓取所有启用的新闻源（海外服务器内存约束，逐个执行）
+
+    Args:
+        source_filter: 可选，只抓取指定 source_id
+
+    Returns:
+        PipelineStats 总体统计
+    """
+    sources, _settings = load_sources()
+
+    if source_filter:
+        sources = [s for s in sources if s.id == source_filter]
+        if not sources:
+            raise ValueError(f"未找到启用的源: {source_filter}")
+
+    logger.info("══════ 开始串行抓取 %d 个新闻源 ══════", len(sources))
+
+    start_time = datetime.now()
+    stats = PipelineStats(start_time=start_time.isoformat(), sources_crawled=0)
+
+    # 串行执行每个源
+    for source in sources:
+        result = await _crawl_single_source_with_storage(source)
+
+        if isinstance(result, Exception):
+            logger.error("源抓取异常: %s", result)
+            stats.sources_failed += 1
+        else:
+            stats.sources_crawled += 1
+            stats.total_articles += result.total_success
+            stats.results.append(result)
+            if result.error:
+                stats.sources_failed += 1
+
+    stats.end_time = datetime.now().isoformat()
+    elapsed = (datetime.now() - start_time).total_seconds()
+
+    logger.info(
+        "══════ 抓取完成: %d/%d 源成功，共 %d 篇文章，耗时 %.1f 秒 ══════",
+        stats.sources_crawled - stats.sources_failed,
+        stats.sources_crawled,
+        stats.total_articles,
+        elapsed,
+    )
+
+    return stats
+
+
+def run_crawl_sync(source_filter: str | None = None) -> PipelineStats:
+    """同步包装器，供 CLI 调用"""
+    return asyncio.run(crawl_all_sources(source_filter=source_filter))

crawler/rss_crawler.py

@@ -0,0 +1,393 @@
+"""RSS/Atom Feed 抓取模块。
+
+用于有 RSS 服务的新闻源（如 MarketWatch），绕过网页反爬。
+支持 RSS 2.0、Atom、Google News RSS 三种格式。
+
+Google News RSS: 处理标题去来源后缀、链接为 Google 跳转 URL。
+"""
+
+import hashlib
+import logging
+import re
+from datetime import datetime, timezone
+from html import unescape
+from pathlib import Path
+from typing import Any
+from urllib.parse import urljoin
+from xml.etree import ElementTree
+
+import httpx
+
+from crawler.models import ArticleItem, CrawlResult, SourceConfig
+from crawler.utils import get_news_day
+
+logger = logging.getLogger(__name__)
+
+# ── HTML 标签清洗 ──────────────────────────────────────
+
+
+def _strip_html(text: str) -> str:
+    """去除 HTML 标签，保留纯文本。"""
+    if not text:
+        return ""
+    text = unescape(text)
+    return re.sub(r"<[^>]+>", "", text).strip()
+
+
+# ── Namespace ──────────────────────────────────────────
+
+
+def _ns(tag: str) -> str:
+    """Atom namespace helper。"""
+    return f"{{http://www.w3.org/2005/Atom}}{tag}"
+
+
+def _compute_url_hash(url: str) -> str:
+    return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
+
+
+# 非内容类扩展名 — RSS 项也可能包含静态资源
+_SKIP_EXT = re.compile(
+    r"\.(png|ico|gif|jpg|jpeg|svg|webp|css|js|xml|json|rss|pdf|zip|woff2?|ttf|eot)"
+    r"([?#]|$)",
+    re.IGNORECASE,
+)
+
+# 明显非文章路径
+_SKIP_PATH = re.compile(
+    r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)"
+    r"[/.]",
+    re.IGNORECASE,
+)
+
+
+def _is_valid_article_url(url: str, source: SourceConfig) -> bool:
+    """校验 RSS 条目 URL 是否为有效文章链接。
+
+    过滤规则:
+        1. 静态资源（JS/CSS/图片/字体/数据文件）
+        2. 技术路径（manifest/_next/static/robots）
+        3. 源 article_url_pattern 不匹配
+        4. 域名不一致（RSS 跨站污染，如 Yahoo Finance RSS 混入 sports/tech）
+    """
+    if _SKIP_EXT.search(url):
+        logger.debug("RSS 跳过静态资源: %s", url[:80])
+        return False
+    if _SKIP_PATH.search(url):
+        logger.debug("RSS 跳过技术路径: %s", url[:80])
+        return False
+
+    # 源级 article_url_pattern 校验
+    pattern = getattr(source, "article_url_pattern", "")
+    if pattern:
+        try:
+            if not re.search(pattern, url, re.IGNORECASE):
+                logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80])
+                return False
+        except re.error:
+            pass  # 正则异常不阻塞
+
+    # 域名一致性校验（RSS 跨站污染防护）
+    # 例外: Google News RSS 的链接是 news.google.com 跳转 URL，跳过域名校验
+    rss_url = getattr(source, "rss_url", "")
+    if rss_url and "news.google.com" not in rss_url:
+        from urllib.parse import urlparse
+        item_domain = (urlparse(url).hostname or "").removeprefix("www.")
+        hp_domain = (urlparse(source.homepage).hostname or "").removeprefix("www.")
+
+        if hp_domain and item_domain:
+            if item_domain != hp_domain and not item_domain.endswith("." + hp_domain):
+                logger.debug(
+                    "RSS 跳过跨站 URL: %s (domain=%s, expected=%s)",
+                    url[:80], item_domain, hp_domain,
+                )
+                return False
+
+    return True
+
+
+def _parse_date(date_str: str | None) -> str:
+    """尝试解析常见日期格式，返回 ISO 8601 字符串。"""
+    if not date_str:
+        return datetime.now().isoformat()
+
+    formats = [
+        "%a, %d %b %Y %H:%M:%S %z",   # RFC 2822
+        "%a, %d %b %Y %H:%M:%S %Z",
+        "%Y-%m-%dT%H:%M:%S%z",         # ISO 8601
+        "%Y-%m-%dT%H:%M:%SZ",
+        "%Y-%m-%d %H:%M:%S",
+        "%Y-%m-%d",
+    ]
+    for fmt in formats:
+        try:
+            return datetime.strptime(date_str, fmt).isoformat()
+        except ValueError:
+            continue
+    return date_str
+
+
+# ── Google News RSS 标题清洗 ────────────────────────────
+
+_KNOWN_SOURCES = [
+    "Reuters", "WSJ", "Bloomberg", "CNBC", "Financial Times",
+    "MarketWatch", "Barron's", "Yahoo Finance", "Investing.com",
+    "Seeking Alpha", "The Economist", "ForexLive", "ZeroHedge",
+]
+
+
+def _clean_google_news_title(title: str) -> str:
+    """Google News RSS 标题格式: 'Article Title - SourceName' → 去掉来源后缀。"""
+    for src in _KNOWN_SOURCES:
+        suffix = f" - {src}"
+        if title.endswith(suffix):
+            return title[: -len(suffix)].strip()
+    # 通用回退：最后一个 " - " 之后可能是来源
+    last_dash = title.rfind(" - ")
+    if last_dash > 0:
+        suffix = title[last_dash + 3:]
+        if len(suffix) < 30 and not suffix.startswith("http"):
+            return title[:last_dash].strip()
+    return title
+
+
+# ── RSS/Atom 解析 ─────────────────────────────────────
+
+
+def _extract_rss_items(xml_text: str) -> list[dict[str, Any]]:
+    """从 RSS/Atom XML 中提取文章条目。
+
+    自动识别：RSS 2.0、Google News RSS、Atom。
+    """
+    # 清理可能的 BOM
+    if xml_text.startswith(""):
+        xml_text = xml_text[1:]
+
+    root = ElementTree.fromstring(xml_text)
+    items: list[dict[str, Any]] = []
+
+    # 检测是否为 Google News RSS
+    first_item = root.find(".//item")
+    is_google_news = False
+    if first_item is not None:
+        source_el = first_item.find("source")
+        if source_el is not None and source_el.text:
+            is_google_news = True
+
+    # RSS 2.0 / Google News RSS: channel/item
+    for item in root.iter("item"):
+        title = _strip_html(_text(item, "title"))
+        link = _text(item, "link")
+        description = _strip_html(_text(item, "description"))
+        pub_date = _text(item, "pubDate")
+
+        if is_google_news and title:
+            title = _clean_google_news_title(title)
+            # 描述通常比标题更长，含摘要
+            if description and len(description) > len(title):
+                summary = description
+            else:
+                summary = title
+        else:
+            summary = description or title
+
+        if link and title:
+            items.append({
+                "title": title,
+                "url": link.strip(),
+                "summary": summary,
+                "publish_time": _parse_date(pub_date),
+            })
+
+    if items:
+        tag = "Google News RSS" if is_google_news else "RSS 2.0"
+        logger.debug("%s: 提取 %d 条", tag, len(items))
+        return items
+
+    # Atom: feed/entry
+    for entry in root.iter(_ns("entry")):
+        title = _strip_html(_text(entry, _ns("title")))
+        link = _attr(entry, _ns("link"), "href")
+        summary = _strip_html(_text(entry, _ns("summary")))
+        updated = _text(entry, _ns("updated"))
+        if link and title:
+            items.append({
+                "title": title,
+                "url": link.strip(),
+                "summary": summary or title,
+                "publish_time": _parse_date(updated),
+            })
+
+    logger.debug("Atom: 提取 %d 条", len(items))
+    return items
+
+
+def _text(element: ElementTree.Element, tag: str) -> str:
+    """安全获取子元素文本。"""
+    child = element.find(tag)
+    return (child.text or "").strip() if child is not None and child.text else ""
+
+
+def _attr(element: ElementTree.Element, tag: str, attr: str) -> str:
+    """安全获取子元素属性。"""
+    child = element.find(tag)
+    return (child.get(attr) or "").strip() if child is not None else ""
+
+
+# ── 主抓取接口 ───────────────────────────────────────
+
+
+def crawl_rss_source(source: SourceConfig) -> CrawlResult:
+    """通过 RSS/Atom Feed 抓取单个新闻源。
+
+    支持 Google News RSS 代理模式：
+    - rss_url 为 news.google.com 时自动识别
+    - 标题自动去来源后缀
+    - 不抓取原文（跳过 Crawl4AI），直接用 RSS 摘要入库
+
+    Args:
+        source: 源配置（需含 rss_url 字段）
+
+    Returns:
+        CrawlResult
+    """
+    rss_url = getattr(source, "rss_url", None)
+    if not rss_url:
+        return CrawlResult(
+            source_id=source.id,
+            source_name=source.name,
+            error="源未配置 rss_url",
+        )
+
+    today = get_news_day()
+    out_dir = Path(f"data/raw/{source.id}/{today}")
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    start_time = datetime.now()
+    articles: list[ArticleItem] = []
+    crawl_time = start_time.isoformat()
+
+    try:
+        resp = httpx.get(
+            rss_url,
+            follow_redirects=True,
+            timeout=30.0,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                    "AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"
+                ),
+                "Accept": "application/rss+xml, application/xml, text/xml, */*",
+            },
+        )
+        resp.raise_for_status()
+        xml_text = resp.text
+    except Exception as e:
+        logger.error("[%s] RSS 抓取失败: %s", source.id, e)
+        return CrawlResult(
+            source_id=source.id,
+            source_name=source.name,
+            start_time=crawl_time,
+            end_time=datetime.now().isoformat(),
+            error=str(e),
+        )
+
+    # 解析 RSS
+    try:
+        items = _extract_rss_items(xml_text)
+    except Exception as e:
+        logger.error("[%s] RSS 解析失败: %s", source.id, e)
+        return CrawlResult(
+            source_id=source.id,
+            source_name=source.name,
+            start_time=crawl_time,
+            end_time=datetime.now().isoformat(),
+            error=f"RSS 解析失败: {e}",
+        )
+
+    if not items:
+        logger.warning("[%s] RSS 返回 0 条", source.id)
+        return CrawlResult(
+            source_id=source.id,
+            source_name=source.name,
+            total_found=0,
+            start_time=crawl_time,
+            end_time=datetime.now().isoformat(),
+        )
+
+    # 过滤已存在的 URL（增量）
+    index_path = out_dir / "index.jsonl"
+    existing_hashes: set[str] = _load_existing_hashes(index_path)
+
+    # 构造 ArticleItem
+    success = 0
+    for item in items[:source.max_articles_per_run]:
+        url_hash = _compute_url_hash(item["url"])
+        if url_hash in existing_hashes:
+            logger.debug("[%s] 跳过 RSS 已抓取: %s", source.id, item["url"][:80])
+            continue
+
+        # URL 有效性校验（静态资源 / 跨站污染 / 不匹配 article_url_pattern）
+        if not _is_valid_article_url(item["url"], source):
+            continue
+
+        # 保存文章摘要为 Markdown
+        md_content = f"# {item['title']}\n\n"
+        md_content += f"**来源**: {source.name}\n\n"
+        md_content += f"**发布时间**: {item['publish_time']}\n\n"
+        md_content += f"**原文链接**: {item['url']}\n\n"
+        md_content += f"{item['summary']}\n"
+
+        md_path = out_dir / f"{url_hash}.md"
+        md_path.write_text(md_content, encoding="utf-8")
+        html_path = out_dir / f"{url_hash}.html"
+        html_content = f"<html><head><title>{item['title']}</title></head><body>{item['summary']}</body></html>"
+        html_path.write_text(html_content, encoding="utf-8")
+
+        article = ArticleItem(
+            source_id=source.id,
+            source_name=source.name,
+            url=item["url"],
+            url_hash=url_hash,
+            title=item["title"],
+            crawl_time=crawl_time,
+            publish_time=item["publish_time"],
+            html_path=str(html_path),
+            md_path=str(md_path),
+            word_count=len(item["summary"].split()),
+            status="success",
+        )
+        articles.append(article)
+        existing_hashes.add(url_hash)
+        success += 1
+
+    logger.info("[%s] RSS 抓取: %d 条新文章（总共 %d 条）",
+                source.id, success, len(items))
+
+    return CrawlResult(
+        source_id=source.id,
+        source_name=source.name,
+        total_found=len(items),
+        total_success=success,
+        articles=articles,
+        start_time=crawl_time,
+        end_time=datetime.now().isoformat(),
+    )
+
+
+def _load_existing_hashes(index_path: Path) -> set[str]:
+    """读取 index.jsonl 中已有 url_hash。"""
+    hashes: set[str] = set()
+    if not index_path.exists():
+        return hashes
+    import json
+    with open(index_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                hashes.add(json.loads(line).get("url_hash", ""))
+            except json.JSONDecodeError:
+                continue
+    return hashes

crawler/storage.py

@@ -0,0 +1,95 @@
+"""存储管理：index.jsonl 读写、输出目录管理"""
+
+import json
+import logging
+from pathlib import Path
+
+from crawler.models import ArticleItem, CrawlResult
+from crawler.utils import get_news_day
+
+logger = logging.getLogger(__name__)
+
+
+def write_index_jsonl(result: CrawlResult) -> Path:
+    """将单源抓取结果写入 index.jsonl
+
+    Args:
+        result: 单源抓取结果
+
+    Returns:
+        index 文件路径
+    """
+    today = get_news_day()
+    out_dir = Path(f"data/raw/{result.source_id}/{today}")
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    index_path = out_dir / "index.jsonl"
+
+    # 追加写入（同一天多次抓取合并）
+    existing: set[str] = set()
+    if index_path.exists():
+        with open(index_path, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    item = json.loads(line)
+                    existing.add(item.get("url_hash", ""))
+                except json.JSONDecodeError:
+                    continue
+
+    written = 0
+    with open(index_path, "a", encoding="utf-8") as f:
+        for article in result.articles:
+            if article.status != "success":
+                continue
+            if article.url_hash in existing:
+                continue  # 跳过已存在的
+
+            record = article.model_dump()
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+            existing.add(article.url_hash)
+            written += 1
+
+    logger.info("[%s] index.jsonl 写入 %d 条（跳过重复 %d 条）",
+                result.source_id, written,
+                len(result.articles) - written)
+    return index_path
+
+
+def load_index(
+    source_id: str,
+    date_str: str | None = None,
+) -> list[ArticleItem]:
+    """读取指定源/日期的 index.jsonl
+
+    Args:
+        source_id: 新闻源 ID
+        date_str: 日期字符串 YYYYMMDD，默认当前新闻日
+
+    Returns:
+        ArticleItem 列表
+    """
+    if date_str is None:
+        date_str = get_news_day()
+
+    index_path = Path(f"data/raw/{source_id}/{date_str}/index.jsonl")
+
+    if not index_path.exists():
+        logger.warning("index 文件不存在: %s", index_path)
+        return []
+
+    articles: list[ArticleItem] = []
+    with open(index_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                data = json.loads(line)
+                articles.append(ArticleItem(**data))
+            except (json.JSONDecodeError, Exception) as e:
+                logger.warning("解析 index 行失败: %s", e)
+
+    return articles

crawler/utils.py

@@ -0,0 +1,51 @@
+"""爬虫工具函数"""
+
+import logging
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+# 默认日切分小时（凌晨）
+DEFAULT_CUTOFF_HOUR = 6
+
+
+def _load_cutoff_hour() -> int:
+    """从 system.yaml 读取日切分小时"""
+    config_path = Path("configs/system.yaml")
+    if config_path.exists():
+        try:
+            with open(config_path, encoding="utf-8") as f:
+                raw = yaml.safe_load(f)
+            return int(raw.get("schedule", {}).get("day_cutoff_hour", DEFAULT_CUTOFF_HOUR))
+        except Exception:
+            pass
+    return DEFAULT_CUTOFF_HOUR
+
+
+def get_news_day(now: datetime | None = None) -> str:
+    """获取当前新闻日（YYYYMMDD）
+
+    规则：当天 06:00 到次日 05:59 属于同一个新闻日。
+    例如 2026-06-19 04:00 → "20260618"，07:00 → "20260619"
+
+    Args:
+        now: 参考时间，默认当前时间
+
+    Returns:
+        新闻日字符串 YYYYMMDD
+    """
+    if now is None:
+        now = datetime.now()
+
+    cutoff = _load_cutoff_hour()
+
+    if now.hour < cutoff:
+        # 凌晨 0:00 - 5:59，属于前一天
+        news_date = now - timedelta(days=1)
+    else:
+        news_date = now
+
+    return news_date.strftime("%Y%m%d")