68 lines
2.2 KiB
Python
68 lines
2.2 KiB
Python
"""cninfo 公告抓取入口。
|
|
|
|
用法:
|
|
uv run python -m scripts.run_cninfo # 抓取今天公告
|
|
uv run python -m scripts.run_cninfo --days 3 # 抓取最近 3 天
|
|
uv run python -m scripts.run_cninfo --max-pages 50 # 最多 50 页
|
|
uv run python -m scripts.run_cninfo --no-save # 试跑不存盘
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from datetime import date, timedelta
|
|
from pathlib import Path
|
|
|
|
from dotenv import load_dotenv
|
|
from loguru import logger
|
|
|
|
from crawler.cninfo import crawl_announcements
|
|
|
|
|
|
def _setup_logger(level: str) -> None:
|
|
logger.remove()
|
|
logger.add(sys.stderr, level=level,
|
|
format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}")
|
|
log_path = Path("logs") / "cninfo.log"
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
|
|
|
|
|
|
def main() -> int:
|
|
load_dotenv()
|
|
parser = argparse.ArgumentParser(description="cninfo 巨潮资讯网公告抓取")
|
|
parser.add_argument("--days", type=int, default=0,
|
|
help="抓取最近 N 天的公告,默认读 CNINFO_DAYS_BACK")
|
|
parser.add_argument("--start", default=None, help="开始日期 YYYY-MM-DD")
|
|
parser.add_argument("--end", default=None, help="结束日期 YYYY-MM-DD")
|
|
parser.add_argument("--max-pages", type=int, default=0, help="最大页数")
|
|
parser.add_argument("--no-save", action="store_true", help="仅抓取不存盘")
|
|
parser.add_argument("--log-level", default="INFO")
|
|
args = parser.parse_args()
|
|
|
|
_setup_logger(args.log_level)
|
|
|
|
start = args.start
|
|
end = args.end
|
|
if args.days > 0 and not start:
|
|
start = (date.today() - timedelta(days=args.days)).strftime("%Y-%m-%d")
|
|
if not end:
|
|
end = date.today().strftime("%Y-%m-%d")
|
|
|
|
results = crawl_announcements(
|
|
start_date=start,
|
|
end_date=end,
|
|
max_pages=args.max_pages or None,
|
|
save=not args.no_save,
|
|
)
|
|
|
|
if not results:
|
|
return 1
|
|
logger.info("cninfo 抓取完成: {} 条", len(results))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|