Initial commit

2026-07-18 15:51:01 +08:00
commit f2c80c5a9c
799 changed files with 133475 additions and 0 deletions
@@ -0,0 +1,107 @@
+"""M1 新闻联播 API 抓取脚本。
+
+从 doorcome API /api/xwlbFine/ 获取 AI 精编的新闻联播条目,
+转换为与 Web 抓取兼容的格式(data/raw/xwlb/{date}/),
+供 M2-M6 管道统一处理。
+
+新闻联播晚间播出,始终抓取前一天数据,不依赖 --date 参数。
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+import urllib.request
+from datetime import date, datetime, timedelta
+from pathlib import Path
+
+from loguru import logger
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
+    )
+
+
+def _url_hash(s: str) -> str:
+    return hashlib.sha1(s.encode("utf-8")).hexdigest()[:16]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="新闻联播 API 抓取 (M1-xwlb)")
+    parser.add_argument("--output-root", default="data/raw")
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+
+    # 新闻联播晚间播出,始终抓取前一天
+    day_str = (date.today() - timedelta(days=1)).strftime("%Y%m%d")
+    api_url = f"https://api.doorcome.cn/api/xwlbFine/?start_date={day_str}&end_date={day_str}"
+
+    logger.info("请求新闻联播 API: {}", api_url)
+    try:
+        req = urllib.request.Request(api_url)
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            body = json.loads(resp.read().decode("utf-8"))
+    except Exception as e:
+        logger.error("API 请求失败: {}", e)
+        return 2
+
+    raw_news = body.get("data", {}).get("news", [])
+    if not raw_news:
+        logger.warning("{} 无新闻联播数据", day_str)
+        return 0
+
+    # 准备输出目录
+    out_dir = Path(args.output_root) / "xwlb" / day_str
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    index_path = out_dir / "index.jsonl"
+    saved = 0
+
+    for n in raw_news:
+        sid = n.get("daily_sub_id", 0)
+        title = n.get("news_title", "")
+        content = n.get("news_improve", "")
+        news_day = n.get("news_days", day_str)
+
+        fake_url = f"xwlb://{news_day}/{sid}"
+        h = _url_hash(fake_url)
+        html_file = f"{h}.html"
+
+        html_content = f"""<!DOCTYPE html>
+<html><head><meta charset="utf-8"><title>{title}</title></head>
+<body><article><h1>{title}</h1><div class="article-content">{content}</div></article></body>
+</html>"""
+        (out_dir / f"{h}.html").write_text(html_content, encoding="utf-8")
+
+        meta = {
+            "source_id": "xwlb",
+            "stage": "article",
+            "url": fake_url,
+            "success": True,
+            "status_code": 200,
+            "title": title,
+            "error": None,
+            "fetched_at": datetime.now().isoformat(),
+            "attempts": 1,
+            "url_hash": h,
+            "html_file": html_file,
+        }
+        with index_path.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+        saved += 1
+
+    logger.info("新闻联播 {} 抓取完成: {} 条 -> {}", day_str, saved, out_dir)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())