初始化

crawler/loader.py

@@ -0,0 +1,65 @@
+"""加载和管理新闻源配置"""
+
+import logging
+from pathlib import Path
+
+import yaml
+
+from crawler.models import SourceConfig
+
+logger = logging.getLogger(__name__)
+
+# 默认配置文件路径
+DEFAULT_SOURCES_PATH = Path("configs/sources.yaml")
+
+
+def load_sources(
+    config_path: Path | None = None,
+) -> tuple[list[SourceConfig], dict]:
+    """从 YAML 加载新闻源配置
+
+    Args:
+        config_path: 配置文件路径，默认 configs/sources.yaml
+
+    Returns:
+        (启用的源列表, settings dict)
+    """
+    path = config_path or DEFAULT_SOURCES_PATH
+
+    if not path.exists():
+        raise FileNotFoundError(f"新闻源配置文件不存在: {path}")
+
+    with open(path, encoding="utf-8") as f:
+        raw = yaml.safe_load(f)
+
+    if not raw or "sources" not in raw:
+        raise ValueError(f"配置文件格式错误，缺少 'sources' 字段: {path}")
+
+    settings = raw.get("settings", {})
+    sources: list[SourceConfig] = []
+
+    for item in raw["sources"]:
+        if not isinstance(item, dict):
+            logger.warning("跳过非字典格式的源配置: %s", item)
+            continue
+
+        if not item.get("enabled", True):
+            logger.info("跳过已禁用的源: %s (%s)", item.get("id"), item.get("name"))
+            continue
+
+        try:
+            source = SourceConfig(**item)
+            sources.append(source)
+        except Exception as e:
+            logger.error("解析源配置失败: %s - %s", item.get("id"), e)
+
+    logger.info("成功加载 %d 个启用的新闻源（共 %d 个）", len(sources), len(raw["sources"]))
+    return sources, settings
+
+
+def get_source_by_id(source_id: str, sources: list[SourceConfig]) -> SourceConfig | None:
+    """按 ID 查找源"""
+    for s in sources:
+        if s.id == source_id:
+            return s
+    return None