74 lines
3.0 KiB
Bash
74 lines
3.0 KiB
Bash
#!/bin/bash
|
|
# =============================================
|
|
# 国内服务器：从海外拉取数据并合并到本地 data/raw/
|
|
# =============================================
|
|
# 幂等：多次执行不会产生重复数据
|
|
# 用法：./scripts/domestic_sync.sh [日期]
|
|
# =============================================
|
|
set -euo pipefail
|
|
|
|
# 从 system.yaml 读取配置
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/_load_config.sh"
|
|
|
|
LOCAL_DATA="$LOCAL_PROJECT_DIR/data/raw"
|
|
OVERSEAS_RAW="$CRAWL_PROJECT_DIR/data/raw"
|
|
PACK_TMP="${SYNC_PACK_DIR:-/tmp}"
|
|
|
|
# ── 确定新闻日 ──────────────────────────────────────
|
|
if [ $# -ge 1 ]; then
|
|
NEWS_DAY="$1"
|
|
else
|
|
CUTOFF=6
|
|
HOUR=$(date +%H)
|
|
if [ "$HOUR" -lt "$CUTOFF" ]; then
|
|
NEWS_DAY=$(date -d "yesterday" +%Y%m%d)
|
|
else
|
|
NEWS_DAY=$(date +%Y%m%d)
|
|
fi
|
|
fi
|
|
|
|
PACK_FILE="en_news_${NEWS_DAY}.tar.gz"
|
|
|
|
echo "[$(date)] ═══ 国内同步开始，新闻日: $NEWS_DAY ═══"
|
|
|
|
# ── 1. 增量 rsync ─────────────────────────────────
|
|
echo "[$(date)] [1/4] rsync 增量同步 data/raw/ ..."
|
|
mkdir -p "$LOCAL_DATA"
|
|
rsync -avz --ignore-existing \
|
|
-e "ssh -o ConnectTimeout=10" \
|
|
"$OVERSEAS_HOST:$OVERSEAS_RAW/" \
|
|
"$LOCAL_DATA/" \
|
|
2>&1 | tail -3
|
|
|
|
# ── 2. 拉取压缩包 ─────────────────────────────────
|
|
echo "[$(date)] [2/4] 尝试拉取压缩包 ..."
|
|
PACK_LOCAL="$PACK_TMP/$PACK_FILE"
|
|
|
|
if ssh -o ConnectTimeout=5 "$OVERSEAS_HOST" "[ -f $OVERSEAS_RAW/$PACK_FILE ]" 2>/dev/null; then
|
|
scp "$OVERSEAS_HOST:$OVERSEAS_RAW/$PACK_FILE" "$PACK_LOCAL" 2>/dev/null
|
|
echo "[$(date)] 压缩包拉取成功: $PACK_LOCAL ($(du -h "$PACK_LOCAL" | cut -f1))"
|
|
elif ssh -o ConnectTimeout=5 "$OVERSEAS_HOST" "[ -f /tmp/$PACK_FILE ]" 2>/dev/null; then
|
|
scp "$OVERSEAS_HOST:/tmp/$PACK_FILE" "$PACK_LOCAL" 2>/dev/null
|
|
echo "[$(date)] 压缩包拉取成功 (海外 /tmp): $PACK_LOCAL ($(du -h "$PACK_LOCAL" | cut -f1))"
|
|
else
|
|
echo "[$(date)] 海外无压缩包，跳过"
|
|
fi
|
|
|
|
# ── 3. 解压合并 ───────────────────────────────────
|
|
if [ -f "$PACK_LOCAL" ]; then
|
|
echo "[$(date)] [3/4] 解压合并到 $LOCAL_DATA ..."
|
|
tar xzf "$PACK_LOCAL" -C "$LOCAL_DATA/" --overwrite 2>&1
|
|
echo "[$(date)] 解压完成"
|
|
mkdir -p "$LOCAL_PROJECT_DIR/data/archive"
|
|
mv "$PACK_LOCAL" "$LOCAL_PROJECT_DIR/data/archive/$PACK_FILE" 2>/dev/null || true
|
|
else
|
|
echo "[$(date)] [3/4] 无压缩包，跳过解压"
|
|
fi
|
|
|
|
# ── 4. 验证 ───────────────────────────────────────
|
|
echo "[$(date)] [4/4] 验证..."
|
|
echo "[$(date)] 文件数: $(find "$LOCAL_DATA" -type f ! -name '*.tar.gz' | wc -l)"
|
|
echo "[$(date)] 总大小: $(du -sh "$LOCAL_DATA" 2>/dev/null | cut -f1)"
|
|
echo "[$(date)] ═══ 国内同步完成 ✅ ═══"
|