Initial commit

crawler/__init__.py

@@ -0,0 +1,32 @@
+"""新闻抓取模块 (M1)。
+
+公共 API:
+    - load_crawler_config: 加载 sources.yaml
+    - crawl_all:           一键抓取所有启用的源
+    - crawl_source:        抓取单个源(供测试/调试)
+    - CrawlerConfig / SourceConfig / CrawlResult: 数据模型
+"""
+
+from .config import load_crawler_config
+from .engine import crawl_all, crawl_source, extract_article_links
+from .models import (
+    ArticleLink,
+    CrawlerConfig,
+    CrawlerSettings,
+    CrawlResult,
+    CrawlStage,
+    SourceConfig,
+)
+
+__all__ = [
+    "ArticleLink",
+    "CrawlResult",
+    "CrawlStage",
+    "CrawlerConfig",
+    "CrawlerSettings",
+    "SourceConfig",
+    "crawl_all",
+    "crawl_source",
+    "extract_article_links",
+    "load_crawler_config",
+]

crawler/cninfo.py

@@ -0,0 +1,499 @@
+"""cninfo 巨潮资讯网爬虫 v2.0。
+
+从 watchlist.yaml 的 code + orgId 拼接 URL,通过 Crawl4AI(Playwright)渲染 SPA 页面提取数据。
+三种数据类型:
+  1. 公司最新公告  → https://www.cninfo.com.cn/new/disclosure/stock?stockCode={code}&orgId={orgId}#latestAnnouncement
+  2. 投资者调研    → 同上, #research
+  3. 互动易问答    → https://irm.cninfo.com.cn/ircs/search?keyword={code}
+
+策略:
+  - 公告/调研: Playwright SPA 渲染 → DOM 提取(A 方案,REST API 的 stock 参数不可靠)
+  - 互动易: requests 优先,失败回退 Playwright
+  - 增量: ann_id 去重,首次从 2026-01-01 全量,后续增量 7 天
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import re
+import time
+from datetime import date, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+import requests
+from bs4 import BeautifulSoup
+from loguru import logger
+
+from .models import CninfoItem
+
+# --------------------------------------------------------------------------- #
+# 常量
+# --------------------------------------------------------------------------- #
+
+SOURCE_ID = "cninfo"
+CNINFO_PDF_BASE = os.environ.get("CNINFO_PDF_BASE", "http://static.cninfo.com.cn")
+
+# 日期范围: 从 2026-01-01 开始
+DATE_START = "2026-01-01"
+
+# 翻页限制
+MAX_ANNOUNCE_PAGES = 5
+MAX_RESEARCH_PAGES = 1
+MAX_IRM_ITEMS = 20
+
+# 请求间隔(秒)
+REQUEST_DELAY = 0.5
+
+
+# --------------------------------------------------------------------------- #
+# 工具函数
+# --------------------------------------------------------------------------- #
+
+def _url_hash(url: str) -> str:
+    return hashlib.sha1(url.encode("utf-8")).hexdigest()[:16]
+
+
+def _today_str() -> str:
+    return date.today().strftime("%Y%m%d")
+
+
+def _load_watchlist() -> list[dict]:
+    import yaml
+    try:
+        with open("configs/watchlist.yaml", encoding="utf-8") as f:
+            data = yaml.safe_load(f) or {}
+        return list(data.get("watchlist") or [])
+    except Exception:
+        logger.exception("加载 watchlist.yaml 失败")
+        return []
+
+
+def _make_api_headers() -> dict[str, str]:
+    return {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                       "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+        "Accept": "text/html,application/xhtml+xml",
+    }
+
+
+def _build_stock_url(code: str, org_id: str) -> str:
+    """拼接 cninfo 个股公告页 URL。"""
+    return f"https://www.cninfo.com.cn/new/disclosure/stock?stockCode={code}&orgId={org_id}"
+
+
+# --------------------------------------------------------------------------- #
+# Crawl4AI SPA 渲染
+# --------------------------------------------------------------------------- #
+
+async def _render_page(url: str, timeout_ms: int = 60000,
+                       delay_ms: int = 20) -> str:
+    """用 Crawl4AI 渲染 SPA 页面,返回 HTML 字符串。"""
+    from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
+
+    bconf = BrowserConfig(headless=True, verbose=False)
+    rconf = CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        page_timeout=timeout_ms,
+        delay_before_return_html=delay_ms,
+    )
+    async with AsyncWebCrawler(config=bconf) as c:
+        result = await c.arun(url=url, config=rconf)
+    return getattr(result, "html", "") or ""
+
+
+# --------------------------------------------------------------------------- #
+# 公告 + 调研: 从渲染后的 SPA 页面 DOM 提取
+# --------------------------------------------------------------------------- #
+
+def _parse_announcement_list(html: str, code: str,
+                              item_type: str = "announcement") -> list[CninfoItem]:
+    """从 cninfo 个股页面的渲染 HTML 中提取公告/调研列表。
+
+    页面结构: 公告列表以 <a> 标签呈现,关键属性:
+      - data-seccode: 股票代码
+      - data-id: 公告 ID
+      - href:  包含 announcementTime / announcementType 等参数
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    items: list[CninfoItem] = []
+    seen: set[str] = set()
+
+    # 查找所有带 data-seccode=code 的公告链接
+    for el in soup.find_all(attrs={"data-seccode": code}):
+        ann_id = (el.get("data-id") or "").strip()
+        if not ann_id or ann_id in seen:
+            continue
+        seen.add(ann_id)
+
+        title = el.get_text(strip=True)
+        if len(title) < 5:
+            continue
+
+        href = el.get("href", "")
+
+        # 从 href 提取发布时间
+        pub_time = ""
+        date_match = re.search(r"announcementTime=(\d{4}-\d{2}-\d{2})", href)
+        if date_match:
+            pub_time = date_match.group(1)
+
+        # 公告类型
+        ann_type = ""
+        type_match = re.search(r"announcementType=(\w+)", href)
+        if type_match:
+            ann_type = type_match.group(1)
+
+        # 板块代码
+        plate = ""
+        plate_match = re.search(r"plate=(\w+)", href)
+        if plate_match:
+            plate = plate_match.group(1)
+
+        # PDF URL
+        pdf_url = ""
+        if pub_time and ann_id:
+            pdf_url = f"{CNINFO_PDF_BASE}/finalpage/{pub_time}/{ann_id}.PDF"
+
+        items.append(CninfoItem(
+            stock_code=code,
+            stock_name="",
+            title=title,
+            content="",
+            publish_time=pub_time,
+            item_type=item_type,
+            url=pdf_url,
+            ann_id=ann_id,
+            extra={"announcement_type": ann_type, "plate": plate},
+        ))
+
+    return items
+
+
+def _parse_irm_text(text: str, code: str) -> list[CninfoItem]:
+    """从互动易页面文本中提取问答。
+
+    注意: cninfo 互动易搜索页是 Vue SPA，问答数据通过需认证的 API 加载。
+    公开访问时页面显示"暂无数据"，此时应返回空列表。
+    """
+    items: list[CninfoItem] = []
+    url = f"https://irm.cninfo.com.cn/ircs/search?keyword={code}"
+
+    # 检测是否为无数据页面
+    if "暂无数据" in text:
+        logger.debug("  {} 互动易页面显示'暂无数据'(需登录认证)", code)
+        return items
+
+    # 检测是否为 SPA 空壳（无 JS 渲染时只有导航文本）
+    text_stripped = text.strip()
+    if len(text_stripped) < 200 and code in text_stripped:
+        logger.debug("  {} 互动易页面内容过短(SPA 空壳)", code)
+        return items
+
+    # 按常见分隔模式拆分问答块
+    blocks = re.split(r"\n(?=\d+\.|\b[问答][：:])", text)
+    for i, block in enumerate(blocks[:MAX_IRM_ITEMS]):
+        block = block.strip()
+        if len(block) > 30 and code in block:
+            items.append(CninfoItem(
+                stock_code=code,
+                stock_name="",
+                title=f"{code} 互动问答 #{i + 1}",
+                content=block[:5000],
+                publish_time="",
+                item_type="irm",
+                url=url,
+                ann_id=_url_hash(f"{url}#{i}"),
+            ))
+    return items
+
+
+# --------------------------------------------------------------------------- #
+# 互动易
+# --------------------------------------------------------------------------- #
+
+async def _fetch_irm_playwright(code: str) -> list[CninfoItem]:
+    """Playwright 渲染互动易搜索页。"""
+    url = f"https://irm.cninfo.com.cn/ircs/search?keyword={code}"
+    try:
+        html = await _render_page(url, timeout_ms=30000, delay_ms=8)
+    except Exception as e:
+        logger.warning("  {} 互动易 Playwright 失败: {}", code, e)
+        return []
+
+    soup = BeautifulSoup(html, "html.parser")
+    body_text = soup.get_text()
+    return _parse_irm_text(body_text, code)
+
+
+def _fetch_irm_requests(code: str) -> list[CninfoItem]:
+    """requests 获取互动易搜索页。"""
+    url = f"https://irm.cninfo.com.cn/ircs/search?keyword={code}"
+    headers = _make_api_headers()
+    headers["Referer"] = "https://irm.cninfo.com.cn/"
+
+    try:
+        r = requests.get(url, headers=headers, timeout=15)
+        r.raise_for_status()
+    except Exception as e:
+        logger.debug("  {} 互动易 requests 失败: {}", code, e)
+        return []
+
+    soup = BeautifulSoup(r.text, "html.parser")
+    body_text = soup.get_text()
+    return _parse_irm_text(body_text, code)
+
+
+# --------------------------------------------------------------------------- #
+# 单股票抓取
+# --------------------------------------------------------------------------- #
+
+async def _crawl_one_stock_async(stock: dict,
+                                 start_date: str = DATE_START,
+                                 end_date: str | None = None) -> list[CninfoItem]:
+    """异步抓取单个公司的公告 + 调研 + 互动易。"""
+    code = stock["code"]
+    name = stock["name"]
+    org_id = stock.get("orgId", "").strip()
+
+    if not org_id:
+        logger.warning("{} ({}) 未配置 orgId,跳过", code, name)
+        return []
+
+    end_date = end_date or date.today().strftime("%Y-%m-%d")
+    base_url = _build_stock_url(code, org_id)
+    results: list[CninfoItem] = []
+
+    # --- 1) 公告 (#latestAnnouncement) ---
+    announce_url = f"{base_url}#latestAnnouncement"
+    logger.info("抓取 {} ({}) 公告: {}", code, name, announce_url[:80])
+    try:
+        html = await _render_page(announce_url, timeout_ms=60000, delay_ms=20)
+        # 日期过滤: 只保留 start_date 之后的
+        items = _parse_announcement_list(html, code, item_type="announcement")
+        filtered = [it for it in items if it.publish_time >= start_date]
+        # 限制页数: 每个页面约 30 条, 5 页 ≈ 150 条(SPA 一次加载可能超过一页)
+        filtered = filtered[:MAX_ANNOUNCE_PAGES * 30]
+        for it in filtered:
+            it.stock_name = name
+        results.extend(filtered)
+        logger.info("  {} 公告: {} 条 (过滤后)", code, len(filtered))
+    except Exception as e:
+        logger.error("  {} 公告抓取失败: {}", code, e)
+
+    # --- 2) 调研 (#research) ---
+    research_url = f"{base_url}#research"
+    logger.info("抓取 {} ({}) 调研: {}", code, name, research_url[:80])
+    try:
+        html = await _render_page(research_url, timeout_ms=60000, delay_ms=20)
+        items = _parse_announcement_list(html, code, item_type="research")
+        filtered = [it for it in items if it.publish_time >= start_date]
+        filtered = filtered[:MAX_RESEARCH_PAGES * 30]
+        for it in filtered:
+            it.stock_name = name
+        results.extend(filtered)
+        logger.info("  {} 调研: {} 条 (过滤后)", code, len(filtered))
+    except Exception as e:
+        logger.warning("  {} 调研抓取失败(可能无调研页面): {}", code, e)
+
+    # --- 3) 互动易 ---
+    logger.info("抓取 {} ({}) 互动易", code, name)
+    try:
+        irm_items = _fetch_irm_requests(code)
+        if not irm_items:
+            logger.info("  {} 互动易 requests 无结果,回退 Playwright...", code)
+            irm_items = await _fetch_irm_playwright(code)
+        for it in irm_items:
+            it.stock_name = name
+        results.extend(irm_items)
+        logger.info("  {} 互动易: {} 条", code, len(irm_items))
+    except Exception as e:
+        logger.error("  {} 互动易抓取失败: {}", code, e)
+
+    return results
+
+
+# --------------------------------------------------------------------------- #
+# 增量保存
+# --------------------------------------------------------------------------- #
+
+def _load_seen_ids(out_dir: Path) -> set[str]:
+    """从 index.jsonl 加载已保存的公告 ID 集合。"""
+    seen: set[str] = set()
+    index_path = out_dir / "index.jsonl"
+    if index_path.is_file():
+        with index_path.open("r", encoding="utf-8") as f:
+            for line in f:
+                try:
+                    rec = json.loads(line.strip())
+                    aid = rec.get("ann_id", "")
+                    if aid:
+                        seen.add(aid)
+                except (json.JSONDecodeError, KeyError):
+                    continue
+    return seen
+
+
+def _save_items(items: list[CninfoItem], out_dir: Path) -> int:
+    """增量保存 CninfoItem 列表,跳过已存在的 ann_id。返回新增条数。"""
+    out_dir.mkdir(parents=True, exist_ok=True)
+    seen = _load_seen_ids(out_dir)
+    logger.info("cninfo 增量模式: 已有 {} 条历史记录", len(seen))
+
+    index_path = out_dir / "index.jsonl"
+    new_count = 0
+
+    with index_path.open("a", encoding="utf-8") as index_f:
+        for item in items:
+            if item.ann_id and item.ann_id in seen:
+                continue
+            seen.add(item.ann_id)
+            new_count += 1
+
+            fname = _url_hash(item.ann_id or item.title + item.stock_code)
+            json_path = out_dir / f"{fname}.json"
+            json_path.write_text(
+                item.model_dump_json(indent=2, ensure_ascii=False),
+                encoding="utf-8",
+            )
+
+            meta = item.model_dump(mode="json")
+            meta["source_id"] = SOURCE_ID
+            meta["json_file"] = f"{fname}.json"
+            index_f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+
+    logger.info("cninfo 已保存: 新增 {} 条 (总计 {} 条) -> {}", new_count, len(seen), out_dir)
+    return new_count
+
+
+# --------------------------------------------------------------------------- #
+# 主入口
+# --------------------------------------------------------------------------- #
+
+def crawl_watchlist(*, save: bool = True) -> list[CninfoItem]:
+    """从 watchlist 抓取所有公司的公告+调研+互动易。
+
+    首次运行从 2026-01-01 开始,后续增量运行(最近 7 天)。
+    通过判定 data/raw/cninfo/ 下是否有历史 index.jsonl 来区分首次/增量。
+    """
+    watchlist = _load_watchlist()
+    if not watchlist:
+        logger.warning("关注列表为空")
+        return []
+
+    out_dir = Path("data/raw") / SOURCE_ID / _today_str()
+
+    # 判断首次还是增量
+    has_history = False
+    raw_root = Path("data/raw") / SOURCE_ID
+    if raw_root.is_dir():
+        for d in sorted(raw_root.glob("*"), reverse=True):
+            idx = d / "index.jsonl"
+            if idx.is_file() and idx.stat().st_size > 0:
+                has_history = True
+                break
+
+    if has_history:
+        start = (date.today() - timedelta(days=7)).strftime("%Y-%m-%d")
+        logger.info("cninfo 增量模式: 日期范围 {} ~ 今天", start)
+    else:
+        start = DATE_START
+        logger.info("cninfo 首次全量: 日期范围 {} ~ 今天", start)
+
+    end = date.today().strftime("%Y-%m-%d")
+
+    # 并发抓取所有股票
+    async def _run_all():
+        tasks = [_crawl_one_stock_async(s, start_date=start, end_date=end)
+                 for s in watchlist]
+        all_items: list[CninfoItem] = []
+        for coro in asyncio.as_completed(tasks):
+            try:
+                items = await coro
+                all_items.extend(items)
+            except Exception as e:
+                logger.error("某股票抓取异常: {}", e)
+        return all_items
+
+    all_items = asyncio.run(_run_all())
+
+    # 统计
+    ann_count = sum(1 for it in all_items if it.item_type == "announcement")
+    res_count = sum(1 for it in all_items if it.item_type == "research")
+    irm_count = sum(1 for it in all_items if it.item_type == "irm")
+    logger.info(
+        "cninfo 抓取完成: 公告 {} / 调研 {} / 互动易 {} ({} 家公司,共 {} 条)",
+        ann_count, res_count, irm_count, len(watchlist), len(all_items),
+    )
+
+    if save and all_items:
+        new_count = _save_items(all_items, out_dir)
+        logger.info("cninfo 本轮新增 {} 条", new_count)
+
+    return all_items
+
+
+# --------------------------------------------------------------------------- #
+# PDF 正文提取
+# --------------------------------------------------------------------------- #
+
+def enrich_articles_with_pdf(day_str: str | None = None, *, limit: int = 0) -> int:
+    """下载 PDF 并用 MarkItDown 提取正文补充到 Article.content。"""
+    day_str = day_str or _today_str()
+    proc_dir = Path("data/processed") / SOURCE_ID / day_str
+    if not proc_dir.is_dir():
+        return 0
+
+    from markitdown import MarkItDown
+
+    enriched = 0
+    for fp in sorted(proc_dir.glob("*.json")):
+        if limit and enriched >= limit:
+            break
+        try:
+            article_data = json.loads(fp.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            continue
+
+        content = article_data.get("content", "")
+        pdf_url = ""
+
+        # 多种方式提取 PDF URL
+        pdf_match = re.search(r"原文链接:\s*(https?://\S+\.pdf)", content, re.IGNORECASE)
+        if pdf_match:
+            pdf_url = pdf_match.group(1)
+        else:
+            pdf_match = re.search(r"PDF链接:\s*(https?://\S+\.pdf)", content, re.IGNORECASE)
+            if pdf_match:
+                pdf_url = pdf_match.group(1)
+            else:
+                url = article_data.get("url", "")
+                if url.lower().endswith(".pdf"):
+                    pdf_url = url
+
+        if not pdf_url:
+            continue
+
+        try:
+            r = requests.get(pdf_url, headers=_make_api_headers(), timeout=30)
+            r.raise_for_status()
+            tmp_path = Path("/tmp") / f"cninfo_pdf_{_url_hash(pdf_url)}.pdf"
+            tmp_path.write_bytes(r.content)
+            md = MarkItDown()
+            result = md.convert(str(tmp_path))
+            text = (result.text_content or "").strip()[:20000]
+            if tmp_path.exists():
+                tmp_path.unlink()
+            if text and len(text) >= 50:
+                article_data["content"] = text
+                article_data["word_count"] = len(text)
+                fp.write_text(json.dumps(article_data, ensure_ascii=False, indent=2),
+                            encoding="utf-8")
+                enriched += 1
+        except Exception as e:
+            logger.warning("PDF 提取失败 {}: {}", pdf_url, e)
+
+    return enriched

crawler/config.py

@@ -0,0 +1,52 @@
+"""抓取配置加载器。
+
+从 YAML 文件读取 sources.yaml,并校验为 CrawlerConfig。
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from loguru import logger
+
+from .models import CrawlerConfig
+
+DEFAULT_CONFIG_PATH = Path("configs/sources.yaml")
+
+
+def load_crawler_config(path: str | Path | None = None) -> CrawlerConfig:
+    """加载并校验抓取配置。
+
+    参数:
+        path: YAML 配置路径,默认 configs/sources.yaml。
+
+    返回:
+        CrawlerConfig 实例。
+
+    异常:
+        FileNotFoundError: 配置文件不存在。
+        ValueError: YAML 顶层不是 mapping 或 sources 缺失。
+        pydantic.ValidationError: schema 校验失败。
+    """
+    config_path = Path(path) if path else DEFAULT_CONFIG_PATH
+    if not config_path.is_file():
+        raise FileNotFoundError(f"抓取配置文件不存在: {config_path}")
+
+    with config_path.open("r", encoding="utf-8") as f:
+        raw: Any = yaml.safe_load(f)
+
+    if not isinstance(raw, dict):
+        raise ValueError(f"配置文件顶层必须是 mapping,实际类型: {type(raw).__name__}")
+    if "sources" not in raw:
+        raise ValueError("配置文件缺少 sources 字段")
+
+    config = CrawlerConfig.model_validate(raw)
+    logger.info(
+        "已加载抓取配置: 总 {} 个源, 启用 {} 个, 并发上限 {}",
+        len(config.sources),
+        len(config.enabled_sources()),
+        config.settings.concurrency,
+    )
+    return config

crawler/engine.py

@@ -0,0 +1,445 @@
+"""Crawl4AI 异步抓取引擎。
+
+设计:
+    1. 一个 AsyncWebCrawler 实例服务所有源(共用浏览器,降低开销);
+    2. 每个 URL 独立的 CrawlerRunConfig(wait_for/timeout 来自源配置);
+    3. 全局 Semaphore 控制并发(默认 3);
+    4. 每个 URL 用 tenacity 重试(默认 3 次,指数退避);
+    5. 两阶段: 先抓首页 -> 解析文章链接 -> 并发抓文章详情。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+from typing import Any
+from urllib.parse import urljoin, urlparse
+
+import httpx
+from bs4 import BeautifulSoup
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
+from loguru import logger
+from markdownify import markdownify as _md_convert
+
+from .models import (
+    ArticleLink,
+    CrawlerConfig,
+    CrawlerSettings,
+    CrawlResult,
+    CrawlStage,
+    SourceConfig,
+)
+from .storage import mark_url_seen, save_result
+
+# --------------------------------------------------------------------------- #
+# 工具函数
+# --------------------------------------------------------------------------- #
+
+def _markdown_text(crawl4ai_markdown: Any) -> str:
+    """从 Crawl4AI 的 markdown 字段中安全取出字符串。
+
+    新版 Crawl4AI(0.4+) 返回 MarkdownGenerationResult 对象;
+    旧版可能直接是 str。这里做兼容。
+    """
+    if crawl4ai_markdown is None:
+        return ""
+    if isinstance(crawl4ai_markdown, str):
+        return crawl4ai_markdown
+    raw = getattr(crawl4ai_markdown, "raw_markdown", None)
+    if isinstance(raw, str):
+        return raw
+    fit = getattr(crawl4ai_markdown, "fit_markdown", None)
+    if isinstance(fit, str):
+        return fit
+    return str(crawl4ai_markdown)
+
+
+def extract_article_links(
+    html: str,
+    base_url: str,
+    source: SourceConfig,
+) -> list[ArticleLink]:
+    """从列表页 HTML 中抽取文章链接,使用源配置的正则筛选。
+
+    步骤:
+        1. 若设置了 article_link_selector,先 narrow 到选择器范围;
+        2. 找出所有 <a href>;
+        3. urljoin 转绝对 URL;
+        4. 用 article_url_pattern 正则筛选;
+        5. 去重保持顺序;
+        6. 截断到 max_articles_per_run。
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    scope = (
+        soup.select_one(source.article_link_selector)
+        if source.article_link_selector
+        else soup
+    )
+    if scope is None:
+        logger.warning(
+            "源 {} 设置了 article_link_selector={!r},但页面中未匹配到", source.id, source.article_link_selector
+        )
+        return []
+
+    pattern = re.compile(source.article_url_pattern)
+    seen: set[str] = set()
+    links: list[ArticleLink] = []
+
+    for a in scope.find_all("a", href=True):
+        href = a["href"].strip()
+        if not href or href.startswith(("javascript:", "#", "mailto:")):
+            continue
+        absolute = urljoin(base_url, href)
+        # 去掉 fragment
+        absolute = absolute.split("#", 1)[0]
+        if not pattern.match(absolute):
+            continue
+        if absolute in seen:
+            continue
+        seen.add(absolute)
+        anchor = (a.get_text() or "").strip() or None
+        links.append(ArticleLink(source_id=source.id, url=absolute, anchor_text=anchor))
+        if len(links) >= source.max_articles_per_run:
+            break
+
+    logger.info(
+        "源 {} 列表页抽取链接 {} 条 (上限 {})",
+        source.id,
+        len(links),
+        source.max_articles_per_run,
+    )
+    return links
+
+
+# --------------------------------------------------------------------------- #
+# 单 URL 抓取
+# --------------------------------------------------------------------------- #
+
+def _build_run_config(source: SourceConfig) -> CrawlerRunConfig:
+    """根据源配置构造 CrawlerRunConfig。"""
+    return CrawlerRunConfig(
+        cache_mode=CacheMode.BYPASS,
+        wait_for=source.wait_for,
+        page_timeout=source.page_timeout_ms,
+        excluded_tags=["script", "style", "noscript"],
+    )
+
+
+class _TimeoutGuard:
+    """单源连续超时计数器，达到阈值后标记跳过，防止整个进程被拖死。"""
+
+    def __init__(self, source_id: str, max_consecutive: int = 2) -> None:
+        self.source_id = source_id
+        self.max_consecutive = max_consecutive
+        self._count = 0
+        self.skip = False
+
+    def record(self, error: str | None, success: bool) -> None:
+        """根据抓取结果更新计数器。"""
+        if self.skip:
+            return
+        if success:
+            self._count = 0
+            return
+        if error and "timeout" in error.lower():
+            self._count += 1
+            if self._count >= self.max_consecutive:
+                self.skip = True
+                logger.warning(
+                    "源 {} 连续超时 {} 次，跳过剩余请求",
+                    self.source_id,
+                    self._count,
+                )
+
+    def skipped_result(self, url: str, stage: CrawlStage) -> CrawlResult:
+        """生成"已跳过"结果。"""
+        return CrawlResult(
+            source_id=self.source_id,
+            stage=stage,
+            url=url,
+            success=False,
+            error="Skipped: 源连续超时已跳过",
+            attempts=0,
+        )
+
+
+async def _fetch_static(
+    url: str,
+    source: SourceConfig,
+    stage: CrawlStage,
+    attempt: int,
+) -> CrawlResult:
+    """使用 httpx 直连抓取静态页面（js_render=False），绕过 Playwright 反爬检测。"""
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+        ),
+    }
+    timeout_sec = source.page_timeout_ms / 1000.0
+    try:
+        async with httpx.AsyncClient(
+            timeout=timeout_sec,
+            follow_redirects=True,
+            headers=headers,
+        ) as client:
+            r = await client.get(url)
+        html = r.text
+        # HTML → Markdown：先清洗再转换
+        markdown = ""
+        if html:
+            try:
+                soup = BeautifulSoup(html, "html.parser")
+                for tag in soup(["script", "style", "noscript", "meta", "link"]):
+                    tag.decompose()
+                markdown = _md_convert(str(soup)) or ""
+            except Exception:
+                markdown = ""
+        return CrawlResult(
+            source_id=source.id,
+            stage=stage,
+            url=url,
+            success=True,
+            status_code=r.status_code,
+            html=html,
+            markdown=markdown,
+            attempts=attempt,
+        )
+    except Exception as e:
+        logger.warning("静态抓取异常 {} {}: {}", source.id, url, e)
+        return CrawlResult(
+            source_id=source.id,
+            stage=stage,
+            url=url,
+            success=False,
+            error=f"{type(e).__name__}: {e}",
+            attempts=attempt,
+        )
+
+
+async def _crawl_once(
+    crawler: AsyncWebCrawler,
+    url: str,
+    source: SourceConfig,
+    stage: CrawlStage,
+    attempt: int,
+) -> CrawlResult:
+    """单次抓取(无重试),用于被 retry 循环包裹。
+
+    - js_render=False 且无 wait_for → httpx 直连,绕过浏览器反爬;
+    - 其他情况 → Playwright。
+    """
+    # 静态页面走 httpx 直连
+    if not source.js_render and not source.wait_for:
+        return await _fetch_static(url, source, stage, attempt)
+
+    # 动态页面走 Playwright
+    run_config = _build_run_config(source)
+    try:
+        c4_result = await crawler.arun(url=url, config=run_config)
+    except Exception as e:
+        logger.warning("抓取异常 {} {}: {}", source.id, url, e)
+        return CrawlResult(
+            source_id=source.id,
+            stage=stage,
+            url=url,
+            success=False,
+            error=f"{type(e).__name__}: {e}",
+            attempts=attempt,
+        )
+
+    success = bool(getattr(c4_result, "success", False))
+    return CrawlResult(
+        source_id=source.id,
+        stage=stage,
+        url=url,
+        success=success,
+        status_code=getattr(c4_result, "status_code", None),
+        html=getattr(c4_result, "html", "") or "",
+        markdown=_markdown_text(getattr(c4_result, "markdown", None)),
+        error=None if success else (getattr(c4_result, "error_message", None) or "unknown"),
+        attempts=attempt,
+    )
+
+
+async def crawl_url_with_retry(
+    crawler: AsyncWebCrawler,
+    url: str,
+    source: SourceConfig,
+    stage: CrawlStage,
+    settings: CrawlerSettings,
+    semaphore: asyncio.Semaphore,
+    guard: _TimeoutGuard | None = None,
+) -> CrawlResult:
+    """带并发限制与指数退避重试的单 URL 抓取。
+
+    重试策略:
+        最多 settings.retry_max_attempts 次;
+        失败 (success=False) 触发重试,等待时间 min(min_wait * 2^(n-1), max_wait)。
+
+    若传入 guard,每次重试前检查是否已被跳过。
+    """
+    async with semaphore:
+        # 检查是否已被跳过
+        if guard and guard.skip:
+            return guard.skipped_result(url, stage)
+
+        max_attempts = settings.retry_max_attempts
+        result: CrawlResult | None = None
+
+        for attempt_no in range(1, max_attempts + 1):
+            result = await _crawl_once(crawler, url, source, stage, attempt_no)
+            if result.success:
+                if guard:
+                    guard.record(None, True)
+                return result
+            if attempt_no >= max_attempts:
+                break
+            # 重试前检查是否已被其他并发请求触发跳过
+            if guard and guard.skip:
+                return guard.skipped_result(url, stage)
+            wait = min(
+                settings.retry_min_wait_sec * (2 ** (attempt_no - 1)),
+                settings.retry_max_wait_sec,
+            )
+            logger.info(
+                "源 {} {} 第 {}/{} 次失败,{} 秒后重试: {}",
+                source.id,
+                url,
+                attempt_no,
+                max_attempts,
+                wait,
+                result.error,
+            )
+            await asyncio.sleep(wait)
+
+        # 此处 result 必非 None(循环至少跑一次)
+        assert result is not None
+        if guard:
+            guard.record(result.error, False)
+        return result
+
+
+# --------------------------------------------------------------------------- #
+# 抓取流程
+# --------------------------------------------------------------------------- #
+
+async def crawl_source(
+    crawler: AsyncWebCrawler,
+    source: SourceConfig,
+    settings: CrawlerSettings,
+    semaphore: asyncio.Semaphore,
+    save: bool = True,
+) -> list[CrawlResult]:
+    """抓取单个新闻源:遍历所有入口 → 抽取链接(去重) → 并发抓文章。"""
+    homepages = source.all_homepages()
+    logger.info(
+        "===== 开始抓取源: {} ({}) {} 个入口 =====",
+        source.id, source.name, len(homepages),
+    )
+
+    all_list_results: list[CrawlResult] = []
+    all_links: list[ArticleLink] = []
+
+    # 增量:加载历史 URL hash
+    from .storage import load_seen_urls
+    from .storage import url_hash as storage_url_hash
+    seen_hashes = load_seen_urls(source.id)
+    if seen_hashes:
+        logger.info("源 {} 增量模式: 已记录 {} 条历史 URL", source.id, len(seen_hashes))
+
+    # 遍历每个入口
+    for hp_url in homepages:
+        list_result = await crawl_url_with_retry(
+            crawler=crawler, url=hp_url, source=source,
+            stage=CrawlStage.LIST, settings=settings, semaphore=semaphore,
+        )
+        if save:
+            save_result(list_result, settings.output_root)
+        all_list_results.append(list_result)
+
+        if not list_result.success:
+            logger.warning("源 {} 入口 {} 抓取失败,跳过: {}", source.id, hp_url, list_result.error)
+            continue
+
+        parsed_base = urlparse(hp_url)
+        base = f"{parsed_base.scheme}://{parsed_base.netloc}"
+        links = extract_article_links(list_result.html, base, source)
+
+        # 去重 + 增量:按 hash 去重(当前运行跨入口 + 历史记录)
+        new_links = [ln for ln in links if storage_url_hash(ln.url) not in seen_hashes]
+        for ln in new_links:
+            seen_hashes.add(storage_url_hash(ln.url))
+        all_links.extend(new_links)
+        # 旧链接数(跨入口去重前)
+        old_count = len(links) - len(new_links)
+        logger.info("源 {} 入口 {} 抽取链接 {} 条(去重后新增 {}, 跳过旧 {} 条)",
+                     source.id, hp_url, len(links), len(new_links), old_count)
+
+    if not all_links:
+        logger.warning("源 {} 所有入口未发现任何文章链接", source.id)
+        return all_list_results
+
+    # 连续超时保护：同源连续超时 2 次后跳过剩余请求
+    guard = _TimeoutGuard(source.id)
+
+    article_tasks = [
+        crawl_url_with_retry(
+            crawler=crawler, url=link.url, source=source,
+            stage=CrawlStage.ARTICLE, settings=settings, semaphore=semaphore,
+            guard=guard,
+        )
+        for link in all_links
+    ]
+    article_results = await asyncio.gather(*article_tasks, return_exceptions=False)
+
+    if save:
+        for r in article_results:
+            save_result(r, settings.output_root)
+
+    success_cnt = sum(1 for r in article_results if r.success)
+    # 记录成功抓取的 URL(增量去重)
+    new_saved = 0
+    for r in article_results:
+        if r.success:
+            mark_url_seen(source.id, r.url)
+            new_saved += 1
+
+    logger.info(
+        "源 {} 完成: 文章 {}/{} 成功, 新增 {} 条已记录",
+        source.id, success_cnt, len(article_results), new_saved,
+    )
+    return [*all_list_results, *article_results]
+
+
+async def crawl_all(
+    config: CrawlerConfig,
+    save: bool = True,
+) -> list[CrawlResult]:
+    """抓取所有启用的源,返回扁平的 CrawlResult 列表。"""
+    settings = config.settings
+    enabled = config.enabled_sources()
+    if not enabled:
+        logger.warning("没有启用的新闻源,直接返回")
+        return []
+
+    browser_config = BrowserConfig(
+        headless=settings.headless,
+        user_agent=settings.user_agent,
+        verbose=False,
+    )
+    semaphore = asyncio.Semaphore(settings.concurrency)
+    all_results: list[CrawlResult] = []
+
+    async with AsyncWebCrawler(config=browser_config) as crawler:
+        for source in enabled:
+            try:
+                results = await crawl_source(crawler, source, settings, semaphore, save=save)
+                all_results.extend(results)
+            except Exception as e:  # 单源异常不应中断整体抓取
+                logger.exception("源 {} 抓取异常: {}", source.id, e)
+
+    total = len(all_results)
+    succ = sum(1 for r in all_results if r.success)
+    logger.info("全部完成: {}/{} 成功 (成功率 {:.0%})", succ, total, succ / max(total, 1))
+    return all_results

crawler/models.py

@@ -0,0 +1,161 @@
+"""新闻抓取模块的数据模型与配置模型。
+
+所有核心对象统一使用 Pydantic 定义(CLAUDE.md 第十条要求)。
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, Field, HttpUrl, field_validator
+
+
+class CrawlStage(StrEnum):
+    """抓取阶段枚举,区分列表页和文章页。"""
+
+    LIST = "list"        # 列表/首页(用于发现文章链接)
+    ARTICLE = "article"  # 文章详情页
+
+
+class SourceConfig(BaseModel):
+    """单个新闻源的配置。
+
+    映射 configs/sources.yaml 中 sources 列表的每一项。
+    """
+
+    id: str = Field(..., description="新闻源短 ID,用作目录名,仅小写字母与下划线")
+    name: str = Field(..., description="中文名,用于日志展示")
+    enabled: bool = Field(default=True, description="是否启用")
+    homepage: HttpUrl = Field(..., description="新闻列表页/首页 URL")
+    extra_homepages: list[HttpUrl] = Field(
+        default_factory=list,
+        description="额外的列表页 URL(同站多频道时使用),共用同一个 article_url_pattern",
+    )
+
+    def all_homepages(self) -> list[str]:
+        """返回所有入口 URL(主入口 + 额外入口)。"""
+        urls = [str(self.homepage)]
+        urls.extend(str(u) for u in self.extra_homepages)
+        return urls
+
+    # 链接发现规则
+    article_url_pattern: str = Field(
+        ...,
+        description="正则表达式,从首页所有链接中筛选出文章 URL",
+    )
+    article_link_selector: str | None = Field(
+        default=None,
+        description="可选 CSS 选择器,优先在选择器内查找文章链接",
+    )
+
+    # JS 渲染相关
+    js_render: bool = Field(
+        default=True,
+        description="是否需要 JS 渲染(财联社/东方财富等动态站点必须为 true)",
+    )
+    wait_for: str | None = Field(
+        default=None,
+        description="可选 CSS,等待该元素出现后再抓取(JS 渲染时使用)",
+    )
+    page_timeout_ms: int = Field(default=30_000, ge=1_000, description="页面加载超时(毫秒)")
+
+    # GNE 提取提示(可选)
+    body_xpath: str | None = Field(
+        default=None,
+        description="GNE body_xpath 参数,强制指定正文容器 XPath",
+    )
+
+    # 抓取数量限制
+    max_articles_per_run: int = Field(
+        default=20,
+        ge=1,
+        le=200,
+        description="单次运行最多抓取的文章数",
+    )
+
+    @field_validator("id")
+    @classmethod
+    def _validate_id(cls, v: str) -> str:
+        if not v.replace("_", "").isalnum() or not v.islower():
+            raise ValueError("source.id 必须为小写字母/数字/下划线")
+        return v
+
+
+class CrawlerSettings(BaseModel):
+    """全局抓取设置。"""
+
+    concurrency: int = Field(default=3, ge=1, le=20, description="并发抓取上限")
+    retry_max_attempts: int = Field(default=3, ge=1, le=10, description="单 URL 最大重试次数")
+    retry_min_wait_sec: float = Field(default=1.0, ge=0.0, description="重试最小等待秒")
+    retry_max_wait_sec: float = Field(default=10.0, ge=0.0, description="重试最大等待秒")
+    user_agent: str = Field(
+        default=(
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+            "(KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+        ),
+        description="浏览器 User-Agent",
+    )
+    headless: bool = Field(default=True, description="浏览器是否无头")
+    output_root: str = Field(default="data/raw", description="抓取结果根目录")
+
+
+class CrawlerConfig(BaseModel):
+    """整个抓取系统的配置(对应 sources.yaml 顶层)。"""
+
+    settings: CrawlerSettings = Field(default_factory=CrawlerSettings)
+    sources: list[SourceConfig]
+
+    def enabled_sources(self) -> list[SourceConfig]:
+        """返回启用的源,按 id 排序。"""
+        return sorted([s for s in self.sources if s.enabled], key=lambda s: s.id)
+
+
+class CninfoItem(BaseModel):
+    """cninfo 公告/调研/互动易数据条目。
+
+    一条记录对应一条公告、一次调研活动或一组互动问答。
+    """
+
+    stock_code: str = Field(..., description="股票代码,6 位数字")
+    stock_name: str = Field(..., description="公司简称")
+    title: str = Field(..., description="标题")
+    content: str = Field(default="", description="正文/摘要内容")
+    publish_time: str = Field(default="", description="发布时间,格式 YYYY-MM-DD")
+    item_type: str = Field(
+        default="announcement",
+        description="数据类型: announcement(公告)|research(调研)|irm(互动易)",
+    )
+    url: str = Field(default="", description="详情页 URL 或 PDF 链接")
+    ann_id: str = Field(default="", description="公告唯一 ID,用于增量去重")
+    extra: dict = Field(default_factory=dict, description="附加字段(公告类型等)")
+
+
+class ArticleLink(BaseModel):
+    """从列表页发现的文章链接。"""
+
+    source_id: str
+    url: str
+    anchor_text: str | None = None
+
+
+class CrawlResult(BaseModel):
+    """单次抓取结果(列表页或文章页通用)。"""
+
+    source_id: str
+    stage: CrawlStage
+    url: str
+    success: bool
+    status_code: int | None = None
+    title: str | None = None
+    html: str = ""
+    markdown: str = ""
+    error: str | None = None
+    fetched_at: datetime = Field(default_factory=datetime.now)
+    attempts: int = 1
+
+    def short_summary(self) -> str:
+        """生成单行摘要,便于日志输出。"""
+        flag = "OK" if self.success else "FAIL"
+        size = len(self.html)
+        return f"[{flag}] {self.source_id} {self.stage.value} {self.url} html={size}B"

crawler/storage.py

@@ -0,0 +1,98 @@
+"""抓取结果本地保存。
+
+目录结构:
+    {output_root}/{source_id}/{YYYYMMDD}/
+        ├── {url_hash}.html         # 原始 HTML
+        ├── {url_hash}.md           # Crawl4AI 输出的 Markdown
+        └── index.jsonl             # 元数据(每行一条 CrawlResult)
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import date
+from pathlib import Path
+
+from loguru import logger
+
+from .models import CrawlResult
+
+
+def url_hash(url: str) -> str:
+    """对 URL 取 SHA1,取前 16 位作为文件名。"""
+    return hashlib.sha1(url.encode("utf-8")).hexdigest()[:16]
+
+
+def build_output_dir(output_root: str | Path, source_id: str, day: date | None = None) -> Path:
+    """构造保存目录: {output_root}/{source_id}/{YYYYMMDD}/。"""
+    target_day = day or date.today()
+    return Path(output_root) / source_id / target_day.strftime("%Y%m%d")
+
+
+def save_result(
+    result: CrawlResult,
+    output_root: str | Path,
+    day: date | None = None,
+    *,
+    skip_existing: bool = True,
+) -> Path | None:
+    """保存单条抓取结果到本地。
+
+    返回 HTML 文件路径;失败结果只追加 index.jsonl,不写 HTML/MD,返回 None。
+
+    skip_existing=True 时:如果当日该 URL 已保存(HTML 文件已存在),跳过写入,
+    仅记录日志,返回已存在路径。保证增量抓取不会重复保存。
+    """
+    out_dir = build_output_dir(output_root, result.source_id, day)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    h = url_hash(result.url)
+    html_path: Path | None = None
+
+    # 增量去重:如果当日已抓取过同一 URL,跳过
+    existing_html = out_dir / f"{h}.html"
+    if skip_existing and existing_html.exists():
+        logger.debug("跳过重复 URL (今日已抓取): {} -> {}", result.url, h)
+        return existing_html
+
+    if result.success and result.html:
+        html_path = out_dir / f"{h}.html"
+        html_path.write_text(result.html, encoding="utf-8")
+
+        if result.markdown:
+            md_path = out_dir / f"{h}.md"
+            md_path.write_text(result.markdown, encoding="utf-8")
+
+    # 元数据(不含 html/markdown 大字段,避免 jsonl 巨大)
+    meta = result.model_dump(exclude={"html", "markdown"}, mode="json")
+    meta["url_hash"] = h
+    meta["html_file"] = html_path.name if html_path else None
+
+    index_path = out_dir / "index.jsonl"
+    with index_path.open("a", encoding="utf-8") as f:
+        f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+
+    logger.debug("已保存抓取结果: {}", result.short_summary())
+    return html_path
+
+
+def load_seen_urls(source_id: str, output_root: str | Path = "data/raw") -> set[str]:
+    """加载某个源已抓取的所有 URL hash(跨日期累积)。
+
+    读取 {output_root}/{source_id}/seen_urls.txt,每行一个 url_hash。
+    """
+    seen_path = Path(output_root) / source_id / "seen_urls.txt"
+    if not seen_path.is_file():
+        return set()
+    with seen_path.open("r", encoding="utf-8") as f:
+        return {line.strip() for line in f if line.strip()}
+
+
+def mark_url_seen(source_id: str, url: str, output_root: str | Path = "data/raw") -> None:
+    """追加一个已抓取 URL 的 hash 到 seen_urls.txt。"""
+    seen_path = Path(output_root) / source_id / "seen_urls.txt"
+    seen_path.parent.mkdir(parents=True, exist_ok=True)
+    h = url_hash(url)
+    with seen_path.open("a", encoding="utf-8") as f:
+        f.write(h + "\n")