"""M1 新闻联播 API 抓取脚本。
从 doorcome API /api/xwlbFine/ 获取 AI 精编的新闻联播条目,
转换为与 Web 抓取兼容的格式(data/raw/xwlb/{date}/),
供 M2-M6 管道统一处理。
新闻联播晚间播出,始终抓取前一天数据,不依赖 --date 参数。
"""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
import urllib.request
from datetime import date, datetime, timedelta
from pathlib import Path
from loguru import logger
def _setup_logger(level: str) -> None:
logger.remove()
logger.add(
sys.stderr,
level=level,
format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
)
def _url_hash(s: str) -> str:
return hashlib.sha1(s.encode("utf-8")).hexdigest()[:16]
def main() -> int:
parser = argparse.ArgumentParser(description="新闻联播 API 抓取 (M1-xwlb)")
parser.add_argument("--output-root", default="data/raw")
parser.add_argument("--log-level", default="INFO")
args = parser.parse_args()
_setup_logger(args.log_level)
# 新闻联播晚间播出,始终抓取前一天
day_str = (date.today() - timedelta(days=1)).strftime("%Y%m%d")
api_url = f"https://api.doorcome.cn/api/xwlbFine/?start_date={day_str}&end_date={day_str}"
logger.info("请求新闻联播 API: {}", api_url)
try:
req = urllib.request.Request(api_url)
with urllib.request.urlopen(req, timeout=15) as resp:
body = json.loads(resp.read().decode("utf-8"))
except Exception as e:
logger.error("API 请求失败: {}", e)
return 2
raw_news = body.get("data", {}).get("news", [])
if not raw_news:
logger.warning("{} 无新闻联播数据", day_str)
return 0
# 准备输出目录
out_dir = Path(args.output_root) / "xwlb" / day_str
out_dir.mkdir(parents=True, exist_ok=True)
index_path = out_dir / "index.jsonl"
saved = 0
for n in raw_news:
sid = n.get("daily_sub_id", 0)
title = n.get("news_title", "")
content = n.get("news_improve", "")
news_day = n.get("news_days", day_str)
fake_url = f"xwlb://{news_day}/{sid}"
h = _url_hash(fake_url)
html_file = f"{h}.html"
html_content = f"""
{title}
{title}
{content}
"""
(out_dir / f"{h}.html").write_text(html_content, encoding="utf-8")
meta = {
"source_id": "xwlb",
"stage": "article",
"url": fake_url,
"success": True,
"status_code": 200,
"title": title,
"error": None,
"fetched_at": datetime.now().isoformat(),
"attempts": 1,
"url_hash": h,
"html_file": html_file,
}
with index_path.open("a", encoding="utf-8") as f:
f.write(json.dumps(meta, ensure_ascii=False) + "\n")
saved += 1
logger.info("新闻联播 {} 抓取完成: {} 条 -> {}", day_str, saved, out_dir)
return 0
if __name__ == "__main__":
raise SystemExit(main())