71 lines
2.7 KiB
Bash
71 lines
2.7 KiB
Bash
#!/bin/bash
|
||
# =============================================
|
||
# 海外服务器：打包压缩当日 data/raw/ 下的所有文件
|
||
# =============================================
|
||
# 用法：./scripts/overseas_pack.sh [日期]
|
||
# 不传日期则使用当前新闻日（基于 day_cutoff_hour）
|
||
# =============================================
|
||
set -euo pipefail
|
||
|
||
# 从 system.yaml 读取配置（有默认值兜底）
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
source "$SCRIPT_DIR/_load_config.sh"
|
||
|
||
PROJECT_DIR="${CRAWL_PROJECT_DIR}"
|
||
DATA_DIR="$PROJECT_DIR/data/raw"
|
||
PACK_TMP="${SYNC_PACK_DIR:-/tmp}"
|
||
SENTINEL_FILE="${SYNC_SENTINEL}"
|
||
|
||
# ── 确定新闻日 ──────────────────────────────────────
|
||
if [ $# -ge 1 ]; then
|
||
NEWS_DAY="$1"
|
||
else
|
||
# 从 system.yaml 读取 cutoff hour
|
||
CUTOFF=$(python3 -c "
|
||
import yaml
|
||
with open('$PROJECT_DIR/configs/system.yaml') as f:
|
||
cfg = yaml.safe_load(f)
|
||
print(cfg.get('schedule', {}).get('day_cutoff_hour', 6))
|
||
" 2>/dev/null || echo 6)
|
||
|
||
HOUR=$(date +%H)
|
||
if [ "$HOUR" -lt "$CUTOFF" ]; then
|
||
NEWS_DAY=$(date -d "yesterday" +%Y%m%d)
|
||
else
|
||
NEWS_DAY=$(date +%Y%m%d)
|
||
fi
|
||
fi
|
||
|
||
echo "[$(date)] 新闻日: $NEWS_DAY"
|
||
|
||
# ── 检查数据是否存在 ─────────────────────────────────
|
||
if [ ! -d "$DATA_DIR" ]; then
|
||
echo "[$(date)] WARNING: data/raw/ 不存在，跳过打包"
|
||
exit 0
|
||
fi
|
||
|
||
# 计算今天目录下的总文件数（排除 tar.gz 和 sentinel）
|
||
TOTAL_FILES=$(find "$DATA_DIR" -path "*/$NEWS_DAY/*" -type f ! -name "*.tar.gz" 2>/dev/null | wc -l)
|
||
if [ "$TOTAL_FILES" -eq 0 ]; then
|
||
echo "[$(date)] WARNING: 新闻日 $NEWS_DAY 无数据文件，跳过打包"
|
||
exit 0
|
||
fi
|
||
|
||
# ── 打包压缩 ─────────────────────────────────────────
|
||
PACK_FILE="$PACK_TMP/en_news_${NEWS_DAY}.tar.gz"
|
||
|
||
echo "[$(date)] 打包 $TOTAL_FILES 个文件 → $PACK_FILE ..."
|
||
|
||
# 收集当天的所有数据文件（包括任意源下面的日期目录）
|
||
find "$DATA_DIR" -path "*/$NEWS_DAY/*" -type f ! -name "*.tar.gz" 2>/dev/null \
|
||
| tar czf "$PACK_FILE" -T - --transform='s|.*/data/raw/||' 2>/dev/null
|
||
|
||
PACK_SIZE=$(du -h "$PACK_FILE" | cut -f1)
|
||
echo "[$(date)] 打包完成: $PACK_FILE ($PACK_SIZE)"
|
||
|
||
# ── 生成哨兵文件 ─────────────────────────────────────
|
||
echo "$NEWS_DAY $(date -Iseconds) $PACK_SIZE ($TOTAL_FILES files)" > "$SENTINEL_FILE"
|
||
echo "[$(date)] 哨兵文件已更新: $SENTINEL_FILE"
|
||
|
||
echo "[$(date)] ✅ 海外打包完成"
|