82 lines
2.7 KiB
Python
82 lines
2.7 KiB
Python
"""M1 抓取入口脚本。
|
|
|
|
用法:
|
|
uv run python -m scripts.run_crawler
|
|
uv run python -m scripts.run_crawler --source cls
|
|
uv run python -m scripts.run_crawler --config configs/sources.yaml --no-save
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
|
from loguru import logger
|
|
|
|
from crawler import crawl_all, crawl_source, load_crawler_config
|
|
|
|
|
|
def _setup_logger(level: str) -> None:
|
|
"""配置 loguru,输出到控制台与 logs/crawler.log。"""
|
|
logger.remove()
|
|
logger.add(sys.stderr, level=level, format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}")
|
|
log_path = Path("logs") / "crawler.log"
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
logger.add(
|
|
log_path,
|
|
level="DEBUG",
|
|
rotation="10 MB",
|
|
retention=5,
|
|
encoding="utf-8",
|
|
enqueue=True,
|
|
)
|
|
|
|
|
|
async def _run(args: argparse.Namespace) -> int:
|
|
config = load_crawler_config(args.config)
|
|
|
|
if args.source:
|
|
source = next((s for s in config.sources if s.id == args.source), None)
|
|
if source is None:
|
|
logger.error("未找到源 id={}", args.source)
|
|
return 2
|
|
if not source.enabled:
|
|
logger.warning("源 {} 在 yaml 中标记为 disabled,本次仍将抓取", source.id)
|
|
|
|
browser_config = BrowserConfig(
|
|
headless=config.settings.headless,
|
|
user_agent=config.settings.user_agent,
|
|
verbose=False,
|
|
)
|
|
sem = asyncio.Semaphore(config.settings.concurrency)
|
|
async with AsyncWebCrawler(config=browser_config) as crawler:
|
|
results = await crawl_source(crawler, source, config.settings, sem, save=not args.no_save)
|
|
else:
|
|
results = await crawl_all(config, save=not args.no_save)
|
|
|
|
succ = sum(1 for r in results if r.success)
|
|
total = len(results)
|
|
rate = succ / max(total, 1)
|
|
logger.info("抓取结束 总 {} 成功 {} 成功率 {:.0%}", total, succ, rate)
|
|
# 验收标准: 成功率 >= 90%
|
|
return 0 if rate >= 0.9 else 1
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="A 股新闻抓取(M1)")
|
|
parser.add_argument("--config", default="configs/sources.yaml", help="sources.yaml 路径")
|
|
parser.add_argument("--source", default=None, help="只抓单个源 id,默认抓全部启用源")
|
|
parser.add_argument("--no-save", action="store_true", help="不写入本地文件,仅试跑")
|
|
parser.add_argument("--log-level", default="INFO", help="DEBUG/INFO/WARNING/ERROR")
|
|
args = parser.parse_args()
|
|
|
|
_setup_logger(args.log_level)
|
|
return asyncio.run(_run(args))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|