初始化

2026-07-18 16:13:52 +08:00
parent c0070f0a5c
commit fe8b417ab6
75 changed files with 12898 additions and 1 deletions
@@ -0,0 +1,183 @@
+#!/bin/bash
+# =============================================
+# 精准修复: 为 publish_time 为空的存量数据补日期
+# =============================================
+# 场景: extractor 升级后，已处理的文章 publish_time 仍为空
+# 策略: 重新提取日期 → 更新 processed JSON → 更新 events JSON
+# 影响: 仅修改 JSON 文件的 publish_time 字段，不触发 LLM 重跑
+# =============================================
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
+cd "$PROJECT_DIR"
+
+echo "[$(date)] ═══ publish_time 精准修复开始 ═══"
+
+.venv/bin/python3 << 'PYEOF'
+import json
+import logging
+from pathlib import Path
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+# ── 复用 extractor 的日期提取（不依赖 HTML 文件存在） ──
+
+def extract_date_from_url(url: str) -> str:
+    """从 URL 提取日期。"""
+    import re
+    from datetime import datetime
+    url_patterns = [
+        r"/(\d{4})/(\d{2})/(\d{2})/",
+        r"/(\d{4})(\d{2})(\d{2})/",
+        r"-(\d{4})(\d{2})(\d{2})(?:[/-]|$)",
+        r"-(\d{4})-(\d{2})-(\d{2})[/-]",
+    ]
+    for pat in url_patterns:
+        m = re.search(pat, url)
+        if m:
+            try:
+                y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
+                return datetime(y, mo, d).isoformat()
+            except ValueError:
+                continue
+    return ""
+
+
+def extract_date_from_html(html_path: str, url: str) -> str:
+    """从 HTML 文件提取日期。"""
+    from extractor.extractor import _extract_publish_time
+    p = Path(html_path)
+    if p.exists():
+        try:
+            html = p.read_text(encoding="utf-8")
+            return _extract_publish_time(html, url=url) or ""
+        except Exception:
+            pass
+    return ""
+
+
+def extract_date_from_md(md_path: str, url: str) -> str:
+    """从 Markdown 文件提取日期（RSS 源在 MD 中写入发布时间）。"""
+    import re
+    p = Path(md_path)
+    if not p.exists():
+        return ""
+    try:
+        md = p.read_text(encoding="utf-8")
+        # RSS 格式: **发布时间**: 2026-06-19T14:30:00
+        m = re.search(r"\*\*发布时间\*\*:?\s*([^\n]+)", md)
+        if m:
+            return m.group(1).strip()
+    except Exception:
+        pass
+    return extract_date_from_url(url)
+
+
+# ════════════════════════════════════════════
+# 阶段 1: 修复 data/processed/ 中的 publish_time
+# ════════════════════════════════════════════
+
+logger.info("═══ 阶段 1: 扫描 data/processed/ ═══")
+fixed_processed = 0
+skipped_ok = 0
+skipped_no_source = 0
+
+for proc_file in Path("data/processed").glob("*/*/*.json"):
+    if proc_file.name == "index.jsonl":
+        continue
+    try:
+        data = json.loads(proc_file.read_text(encoding="utf-8"))
+    except Exception:
+        continue
+
+    # 只处理 publish_time 为空的
+    pt = (data.get("publish_time") or "").strip()
+    if pt:
+        skipped_ok += 1
+        continue
+
+    url = data.get("url", "")
+    html_path = data.get("html_path", "")
+    md_path = data.get("md_path", "")
+
+    # 尝试提取日期
+    new_pt = ""
+    if html_path:
+        new_pt = extract_date_from_html(html_path, url)
+    if not new_pt and md_path:
+        new_pt = extract_date_from_md(md_path, url)
+    if not new_pt:
+        new_pt = extract_date_from_url(url)
+
+    if new_pt:
+        data["publish_time"] = new_pt
+        proc_file.write_text(
+            json.dumps(data, indent=2, ensure_ascii=False),
+            encoding="utf-8",
+        )
+        fixed_processed += 1
+        logger.debug("processed: %s → %s", proc_file.name, new_pt)
+    else:
+        skipped_no_source += 1
+
+logger.info(
+    "阶段 1 完成: 修复 %d, 已有日期 %d, 仍无法提取 %d",
+    fixed_processed, skipped_ok, skipped_no_source,
+)
+
+
+# ════════════════════════════════════════════
+# 阶段 2: 同步修复 data/events/ 中的 publish_time
+# ════════════════════════════════════════════
+
+logger.info("═══ 阶段 2: 扫描 data/events/ ═══")
+fixed_events = 0
+skipped_events_ok = 0
+skipped_events_no_url = 0
+
+for ev_file in Path("data/events").glob("*/*.json"):
+    if ev_file.name == "index.json":
+        continue
+    try:
+        data = json.loads(ev_file.read_text(encoding="utf-8"))
+    except Exception:
+        continue
+
+    pt = (data.get("publish_time") or "").strip()
+    if pt:
+        skipped_events_ok += 1
+        continue
+
+    url = data.get("url", "")
+    new_pt = extract_date_from_url(url)
+
+    if new_pt:
+        data["publish_time"] = new_pt
+        ev_file.write_text(
+            json.dumps(data, indent=2, ensure_ascii=False),
+            encoding="utf-8",
+        )
+        fixed_events += 1
+        logger.debug("events: %s → %s", ev_file.name, new_pt)
+    else:
+        skipped_events_no_url += 1
+
+logger.info(
+    "阶段 2 完成: 修复 %d, 已有日期 %d, 仍无法提取 %d",
+    fixed_events, skipped_events_ok, skipped_events_no_url,
+)
+
+# ── 汇总 ──
+total_fixed = fixed_processed + fixed_events
+total_still_empty = skipped_no_source + skipped_events_no_url
+logger.info("══════ 汇总 ══════")
+logger.info("processes 修复: %d, events 修复: %d", fixed_processed, fixed_events)
+logger.info("总计修复: %d, 仍为空: %d", total_fixed, total_still_empty)
+PYEOF
+
+echo "[$(date)] ═══ publish_time 精准修复完成 ✅ ═══"