feat: 全流程终端实时进度输出 + 步骤日志
- step_run 增加耗时统计(▶ 开始 / ✔ 完成 Ns / ✗ 失败 exit=N)与 STEP_LOG 支持 - domestic_full.sh / pipeline.sh 去掉 tail 截断,输出实时透传终端并 tee 到 logs/*.log - 退出码用 PIPESTATUS[0] 捕获(规避 pipefail/tee 吞码);全角标点后变量用花括号包裹 - 已同步 pi5 实测:M3 31s / M6 20s 耗时显示正常,日志文件生成
This commit is contained in:
@@ -30,26 +30,34 @@ source "$SCRIPT_DIR/_step_state.sh"
|
||||
|
||||
LOG() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||
|
||||
# ── 全流程日志:所有输出实时显示到终端,同时完整写入日志文件 ──
|
||||
LOG_DIR="logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
FULL_LOG="$LOG_DIR/domestic_full_$(date +%Y%m%d_%H%M%S).log"
|
||||
: > "$FULL_LOG"
|
||||
|
||||
LOG "══════ 国内全流程开始(resume=${RESUME})═══════"
|
||||
LOG "全流程日志: ${FULL_LOG}(终端实时显示完整进度)"
|
||||
|
||||
# ── 加载 .env ──
|
||||
export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true)
|
||||
|
||||
# ── 1. M1 Pi 抓取(headful Playwright + HTTP 代理)──
|
||||
# 部分源抓取失败不阻塞管道(原语义);抓取本身按 URL 去重(index.jsonl),
|
||||
# 已抓取过的 URL 不会重复写入
|
||||
# 已抓取过的 URL 不会重复写入;输出实时显示每源进度
|
||||
if step_should_run M1_crawl; then
|
||||
LOG "[1/2] Pi M1 抓取(8G headful + HTTP 代理)..."
|
||||
bash "$SCRIPT_DIR/domestic_crawl_8g.sh" 2>&1 | tail -5 || LOG "WARNING: 部分源抓取失败,继续管道"
|
||||
bash "$SCRIPT_DIR/domestic_crawl_8g.sh" 2>&1 | tee -a "$FULL_LOG" \
|
||||
|| LOG "WARNING: 部分源抓取失败,继续管道"
|
||||
step_mark M1_crawl
|
||||
fi
|
||||
|
||||
# ── 2. M2→M6 管道(含日报)──
|
||||
LOG "[2/2] 全链路管道..."
|
||||
if [ "$RESUME" = "1" ]; then
|
||||
bash "$SCRIPT_DIR/pipeline.sh" --resume 2>&1 | tail -10
|
||||
bash "$SCRIPT_DIR/pipeline.sh" --resume 2>&1 | tee -a "$FULL_LOG"
|
||||
else
|
||||
bash "$SCRIPT_DIR/pipeline.sh" 2>&1 | tail -10
|
||||
bash "$SCRIPT_DIR/pipeline.sh" 2>&1 | tee -a "$FULL_LOG"
|
||||
fi
|
||||
|
||||
LOG "══════ 国内全流程完成 ✅ ══════"
|
||||
|
||||
Reference in New Issue
Block a user