初始化

2026-07-18 16:13:52 +08:00
parent c0070f0a5c
commit fe8b417ab6
75 changed files with 12898 additions and 1 deletions
@@ -0,0 +1,73 @@
+#!/bin/bash
+# =============================================
+# 国内服务器：从海外拉取数据并合并到本地 data/raw/
+# =============================================
+# 幂等：多次执行不会产生重复数据
+# 用法：./scripts/domestic_sync.sh [日期]
+# =============================================
+set -euo pipefail
+
+# 从 system.yaml 读取配置
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/_load_config.sh"
+
+LOCAL_DATA="$LOCAL_PROJECT_DIR/data/raw"
+OVERSEAS_RAW="$CRAWL_PROJECT_DIR/data/raw"
+PACK_TMP="${SYNC_PACK_DIR:-/tmp}"
+
+# ── 确定新闻日 ──────────────────────────────────────
+if [ $# -ge 1 ]; then
+    NEWS_DAY="$1"
+else
+    CUTOFF=6
+    HOUR=$(date +%H)
+    if [ "$HOUR" -lt "$CUTOFF" ]; then
+        NEWS_DAY=$(date -d "yesterday" +%Y%m%d)
+    else
+        NEWS_DAY=$(date +%Y%m%d)
+    fi
+fi
+
+PACK_FILE="en_news_${NEWS_DAY}.tar.gz"
+
+echo "[$(date)] ═══ 国内同步开始，新闻日: $NEWS_DAY ═══"
+
+# ── 1. 增量 rsync ─────────────────────────────────
+echo "[$(date)] [1/4] rsync 增量同步 data/raw/ ..."
+mkdir -p "$LOCAL_DATA"
+rsync -avz --ignore-existing \
+    -e "ssh -o ConnectTimeout=10" \
+    "$OVERSEAS_HOST:$OVERSEAS_RAW/" \
+    "$LOCAL_DATA/" \
+    2>&1 | tail -3
+
+# ── 2. 拉取压缩包 ─────────────────────────────────
+echo "[$(date)] [2/4] 尝试拉取压缩包 ..."
+PACK_LOCAL="$PACK_TMP/$PACK_FILE"
+
+if ssh -o ConnectTimeout=5 "$OVERSEAS_HOST" "[ -f $OVERSEAS_RAW/$PACK_FILE ]" 2>/dev/null; then
+    scp "$OVERSEAS_HOST:$OVERSEAS_RAW/$PACK_FILE" "$PACK_LOCAL" 2>/dev/null
+    echo "[$(date)] 压缩包拉取成功: $PACK_LOCAL ($(du -h "$PACK_LOCAL" | cut -f1))"
+elif ssh -o ConnectTimeout=5 "$OVERSEAS_HOST" "[ -f /tmp/$PACK_FILE ]" 2>/dev/null; then
+    scp "$OVERSEAS_HOST:/tmp/$PACK_FILE" "$PACK_LOCAL" 2>/dev/null
+    echo "[$(date)] 压缩包拉取成功 (海外 /tmp): $PACK_LOCAL ($(du -h "$PACK_LOCAL" | cut -f1))"
+else
+    echo "[$(date)] 海外无压缩包，跳过"
+fi
+
+# ── 3. 解压合并 ───────────────────────────────────
+if [ -f "$PACK_LOCAL" ]; then
+    echo "[$(date)] [3/4] 解压合并到 $LOCAL_DATA ..."
+    tar xzf "$PACK_LOCAL" -C "$LOCAL_DATA/" --overwrite 2>&1
+    echo "[$(date)] 解压完成"
+    mkdir -p "$LOCAL_PROJECT_DIR/data/archive"
+    mv "$PACK_LOCAL" "$LOCAL_PROJECT_DIR/data/archive/$PACK_FILE" 2>/dev/null || true
+else
+    echo "[$(date)] [3/4] 无压缩包，跳过解压"
+fi
+
+# ── 4. 验证 ───────────────────────────────────────
+echo "[$(date)] [4/4] 验证..."
+echo "[$(date)] 文件数: $(find "$LOCAL_DATA" -type f ! -name '*.tar.gz' | wc -l)"
+echo "[$(date)] 总大小: $(du -sh "$LOCAL_DATA" 2>/dev/null | cut -f1)"
+echo "[$(date)] ═══ 国内同步完成 ✅ ═══"