feat: 日报改为每日一次 + 新增 LLM 使用/成本/缓存文档
- domestic_full.sh / pipeline.sh 新增 --no-report 参数:跳过日报生成, 保持 M1 抓取 + M2→M6 管道不变,用于 12:00/18:00 只抓不生成日报 - crontab: 07:00 全量含日报,12:00/18:00 --no-report(日报每日一次,降低 LLM 费用) - docs/llm-models.md: 各 LLM 调用点分别用的 provider/model 事实清单 - docs/llm-cost-analysis.md: 真实 token 消耗与费用构成、降本建议 - docs/llm-cache-optimization.md: DeepSeek 上下文缓存机制与命中率提升设计 - docs/README.md 增加上述三个文档索引
This commit is contained in:
+17
-10
@@ -7,9 +7,11 @@
|
||||
# 每天 06:00 / 12:00 / 18:00 / 22:00 各执行一次
|
||||
# =============================================
|
||||
# 用法:
|
||||
# ./scripts/domestic_full.sh # 全新执行
|
||||
# ./scripts/domestic_full.sh --resume # 从中断处继续(跳过已完成步骤,
|
||||
# # 步骤状态见 data/run_state/{date}.state)
|
||||
# ./scripts/domestic_full.sh # 全新执行(含日报)
|
||||
# ./scripts/domestic_full.sh --resume # 从中断处继续(跳过已完成步骤,
|
||||
# # 步骤状态见 data/run_state/{date}.state)
|
||||
# ./scripts/domestic_full.sh --no-report # 只抓取 + M2→M6,跳过日报生成
|
||||
# ./scripts/domestic_full.sh --no-report --resume
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
@@ -19,10 +21,12 @@ cd "$PROJECT_DIR"
|
||||
|
||||
# ── 参数解析 ──
|
||||
RESUME=0
|
||||
NO_REPORT=0
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--resume) RESUME=1 ;;
|
||||
*) echo "未知参数: ${arg}(支持 --resume)" >&2; exit 1 ;;
|
||||
--no-report) NO_REPORT=1 ;;
|
||||
*) echo "未知参数: ${arg}(支持 --resume / --no-report)" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
@@ -36,7 +40,7 @@ mkdir -p "$LOG_DIR"
|
||||
FULL_LOG="$LOG_DIR/domestic_full_$(date +%Y%m%d_%H%M%S).log"
|
||||
: > "$FULL_LOG"
|
||||
|
||||
LOG "══════ 国内全流程开始(resume=${RESUME})═══════"
|
||||
LOG "══════ 国内全流程开始(resume=${RESUME} no_report=${NO_REPORT})═══════"
|
||||
LOG "全流程日志: ${FULL_LOG}(终端实时显示完整进度)"
|
||||
|
||||
# ── 加载 .env ──
|
||||
@@ -62,13 +66,16 @@ if step_should_run M1_crawl; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── 2. M2→M6 管道(含日报)──
|
||||
LOG "━━━ [2/2] M2→M6 管道 + 日报 ━━━"
|
||||
if [ "$RESUME" = "1" ]; then
|
||||
bash "$SCRIPT_DIR/pipeline.sh" --resume 2>&1 | tee -a "$FULL_LOG"
|
||||
# ── 2. M2→M6 管道(+ 可选日报)──
|
||||
if [ "$NO_REPORT" = "1" ]; then
|
||||
LOG "━━━ [2/2] M2→M6 管道(已跳过日报,--no-report)━━━"
|
||||
else
|
||||
bash "$SCRIPT_DIR/pipeline.sh" 2>&1 | tee -a "$FULL_LOG"
|
||||
LOG "━━━ [2/2] M2→M6 管道 + 日报 ━━━"
|
||||
fi
|
||||
PIPELINE_ARGS=""
|
||||
[ "$RESUME" = "1" ] && PIPELINE_ARGS="$PIPELINE_ARGS --resume"
|
||||
[ "$NO_REPORT" = "1" ] && PIPELINE_ARGS="$PIPELINE_ARGS --no-report"
|
||||
bash "$SCRIPT_DIR/pipeline.sh" $PIPELINE_ARGS 2>&1 | tee -a "$FULL_LOG"
|
||||
|
||||
if [ "$M1_FAILED" -eq 1 ]; then
|
||||
LOG "⚠️ 全流程结束,但 M1 抓取未成功,退出码=1"
|
||||
|
||||
Reference in New Issue
Block a user