intl_news/scripts/fix_publish_time.sh

#!/bin/bash
# =============================================
# 精准修复: 为 publish_time 为空的存量数据补日期
# =============================================
# 场景: extractor 升级后，已处理的文章 publish_time 仍为空
# 策略: 重新提取日期 → 更新 processed JSON → 更新 events JSON
# 影响: 仅修改 JSON 文件的 publish_time 字段，不触发 LLM 重跑
# =============================================
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
cd "$PROJECT_DIR"

echo "[$(date)] ═══ publish_time 精准修复开始 ═══"

.venv/bin/python3 << 'PYEOF'
import json
import logging
from pathlib import Path

logging.basicConfig(
    level=logging.INFO,
    format="%(levelname)s - %(message)s",
)
logger = logging.getLogger(__name__)

# ── 复用 extractor 的日期提取（不依赖 HTML 文件存在） ──

def extract_date_from_url(url: str) -> str:
    """从 URL 提取日期。"""
    import re
    from datetime import datetime
    url_patterns = [
        r"/(\d{4})/(\d{2})/(\d{2})/",
        r"/(\d{4})(\d{2})(\d{2})/",
        r"-(\d{4})(\d{2})(\d{2})(?:[/-]|$)",
        r"-(\d{4})-(\d{2})-(\d{2})[/-]",
    ]
    for pat in url_patterns:
        m = re.search(pat, url)
        if m:
            try:
                y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
                return datetime(y, mo, d).isoformat()
            except ValueError:
                continue
    return ""


def extract_date_from_html(html_path: str, url: str) -> str:
    """从 HTML 文件提取日期。"""
    from extractor.extractor import _extract_publish_time
    p = Path(html_path)
    if p.exists():
        try:
            html = p.read_text(encoding="utf-8")
            return _extract_publish_time(html, url=url) or ""
        except Exception:
            pass
    return ""


def extract_date_from_md(md_path: str, url: str) -> str:
    """从 Markdown 文件提取日期（RSS 源在 MD 中写入发布时间）。"""
    import re
    p = Path(md_path)
    if not p.exists():
        return ""
    try:
        md = p.read_text(encoding="utf-8")
        # RSS 格式: **发布时间**: 2026-06-19T14:30:00
        m = re.search(r"\*\*发布时间\*\*:?\s*([^\n]+)", md)
        if m:
            return m.group(1).strip()
    except Exception:
        pass
    return extract_date_from_url(url)


# ════════════════════════════════════════════
# 阶段 1: 修复 data/processed/ 中的 publish_time
# ════════════════════════════════════════════

logger.info("═══ 阶段 1: 扫描 data/processed/ ═══")
fixed_processed = 0
skipped_ok = 0
skipped_no_source = 0

for proc_file in Path("data/processed").glob("*/*/*.json"):
    if proc_file.name == "index.jsonl":
        continue
    try:
        data = json.loads(proc_file.read_text(encoding="utf-8"))
    except Exception:
        continue

    # 只处理 publish_time 为空的
    pt = (data.get("publish_time") or "").strip()
    if pt:
        skipped_ok += 1
        continue

    url = data.get("url", "")
    html_path = data.get("html_path", "")
    md_path = data.get("md_path", "")

    # 尝试提取日期
    new_pt = ""
    if html_path:
        new_pt = extract_date_from_html(html_path, url)
    if not new_pt and md_path:
        new_pt = extract_date_from_md(md_path, url)
    if not new_pt:
        new_pt = extract_date_from_url(url)

    if new_pt:
        data["publish_time"] = new_pt
        proc_file.write_text(
            json.dumps(data, indent=2, ensure_ascii=False),
            encoding="utf-8",
        )
        fixed_processed += 1
        logger.debug("processed: %s → %s", proc_file.name, new_pt)
    else:
        skipped_no_source += 1

logger.info(
    "阶段 1 完成: 修复 %d, 已有日期 %d, 仍无法提取 %d",
    fixed_processed, skipped_ok, skipped_no_source,
)


# ════════════════════════════════════════════
# 阶段 2: 同步修复 data/events/ 中的 publish_time
# ════════════════════════════════════════════

logger.info("═══ 阶段 2: 扫描 data/events/ ═══")
fixed_events = 0
skipped_events_ok = 0
skipped_events_no_url = 0

for ev_file in Path("data/events").glob("*/*.json"):
    if ev_file.name == "index.json":
        continue
    try:
        data = json.loads(ev_file.read_text(encoding="utf-8"))
    except Exception:
        continue

    pt = (data.get("publish_time") or "").strip()
    if pt:
        skipped_events_ok += 1
        continue

    url = data.get("url", "")
    new_pt = extract_date_from_url(url)

    if new_pt:
        data["publish_time"] = new_pt
        ev_file.write_text(
            json.dumps(data, indent=2, ensure_ascii=False),
            encoding="utf-8",
        )
        fixed_events += 1
        logger.debug("events: %s → %s", ev_file.name, new_pt)
    else:
        skipped_events_no_url += 1

logger.info(
    "阶段 2 完成: 修复 %d, 已有日期 %d, 仍无法提取 %d",
    fixed_events, skipped_events_ok, skipped_events_no_url,
)

# ── 汇总 ──
total_fixed = fixed_processed + fixed_events
total_still_empty = skipped_no_source + skipped_events_no_url
logger.info("══════ 汇总 ══════")
logger.info("processes 修复: %d, events 修复: %d", fixed_processed, fixed_events)
logger.info("总计修复: %d, 仍为空: %d", total_fixed, total_still_empty)
PYEOF

echo "[$(date)] ═══ publish_time 精准修复完成 ✅ ═══"