66 lines
1.8 KiB
Python
66 lines
1.8 KiB
Python
"""加载和管理新闻源配置"""
|
|
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
from crawler.models import SourceConfig
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# 默认配置文件路径
|
|
DEFAULT_SOURCES_PATH = Path("configs/sources.yaml")
|
|
|
|
|
|
def load_sources(
|
|
config_path: Path | None = None,
|
|
) -> tuple[list[SourceConfig], dict]:
|
|
"""从 YAML 加载新闻源配置
|
|
|
|
Args:
|
|
config_path: 配置文件路径，默认 configs/sources.yaml
|
|
|
|
Returns:
|
|
(启用的源列表, settings dict)
|
|
"""
|
|
path = config_path or DEFAULT_SOURCES_PATH
|
|
|
|
if not path.exists():
|
|
raise FileNotFoundError(f"新闻源配置文件不存在: {path}")
|
|
|
|
with open(path, encoding="utf-8") as f:
|
|
raw = yaml.safe_load(f)
|
|
|
|
if not raw or "sources" not in raw:
|
|
raise ValueError(f"配置文件格式错误，缺少 'sources' 字段: {path}")
|
|
|
|
settings = raw.get("settings", {})
|
|
sources: list[SourceConfig] = []
|
|
|
|
for item in raw["sources"]:
|
|
if not isinstance(item, dict):
|
|
logger.warning("跳过非字典格式的源配置: %s", item)
|
|
continue
|
|
|
|
if not item.get("enabled", True):
|
|
logger.info("跳过已禁用的源: %s (%s)", item.get("id"), item.get("name"))
|
|
continue
|
|
|
|
try:
|
|
source = SourceConfig(**item)
|
|
sources.append(source)
|
|
except Exception as e:
|
|
logger.error("解析源配置失败: %s - %s", item.get("id"), e)
|
|
|
|
logger.info("成功加载 %d 个启用的新闻源（共 %d 个）", len(sources), len(raw["sources"]))
|
|
return sources, settings
|
|
|
|
|
|
def get_source_by_id(source_id: str, sources: list[SourceConfig]) -> SourceConfig | None:
|
|
"""按 ID 查找源"""
|
|
for s in sources:
|
|
if s.id == source_id:
|
|
return s
|
|
return None
|