From 012cb615d3773851ab5cc0a53a62ca513d47ba6e Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 12 Aug 2026 11:20:51 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=84=9A=E6=9C=AC=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E6=98=BE=E6=80=A7=E8=BE=93=E5=87=BA=E5=BD=93=E5=89=8D=E9=98=B6?= =?UTF-8?q?=E6=AE=B5=E4=B8=8E=20AI=20=E6=A8=A1=E5=9E=8B=E4=BE=9B=E5=BA=94?= =?UTF-8?q?=E5=95=86/=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pipeline.sh 每阶段 banner（━━━ M4 翻译+事件抽取（AI 大模型: deepseek / deepseek-v4-flash）━━━） - ai_model_info() 从 system.yaml 读取场景模型（translation/daily_report/embedding） - Python 层：M4/M5/日报 显性打印 provider/model（场景标注） - 已同步 pi5 实测：客户端初始化日志含供应商/模型 --- README.md | 10 +++++----- continuation.md | 18 ++++++++++++++++++ embedding/pipeline.py | 4 ++-- llm/pipeline.py | 4 ++++ scheduler/reporter.py | 3 +++ scripts/domestic_full.sh | 4 ++-- scripts/pipeline.sh | 34 +++++++++++++++++++++++++++++++--- 7 files changed, 65 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1f7069d..72949b8 100644 --- a/README.md +++ b/README.md @@ -177,12 +177,12 @@ ls -lt data/reports/ ### 终端进度与日志 -执行全流程时，终端**实时显示完整进度**（每步 `▶ 开始` → 步骤内逐源/逐篇输出 → `✔ 完成（耗时 Ns）`，失败步骤显示 `✗` 与退出码），不再截断： +执行全流程时，终端**实时显示完整进度**，且**显性标注当前阶段与 AI 模型**： -- `logs/domestic_full_{YYYYMMDD_HHMMSS}.log` — 全流程完整日志（M1 + 管道） -- `logs/pipeline_{YYYYMMDD_HHMMSS}.log` — 管道步骤完整日志 - -日志文件已加入 `.gitignore`。 +- 每阶段标题：`━━━ M2 正文提取 ━━━`、`━━━ M4 翻译+事件抽取（AI 大模型: deepseek / deepseek-v4-flash）━━━`（AI 阶段自动从 `system.yaml` 读取供应商/模型） +- 每步进度：`▶ 步骤 开始` → 步骤内逐源/逐篇输出 → `✔ 完成（耗时 Ns）`；失败显示 `✗` 与退出码 +- AI 调用点在 Python 层同样显性打印（`AI 大模型（场景 translation/daily_report）: provider=... model=...`、`初始化 Embedding 客户端: provider=dashscope model=...`） +- 日志：`logs/domestic_full_{ts}.log`（全流程）、`logs/pipeline_{ts}.log`（管道），已入 `.gitignore` ### 日报入库（M9） diff --git a/continuation.md b/continuation.md index 8655a61..f7609b5 100644 --- a/continuation.md +++ b/continuation.md @@ -4,6 +4,24 @@ --- +## 2026-08-12 会话成果（四）：脚本阶段/模型显性输出 + +**背景：** 执行全流程时终端未显性标注"当前阶段"，AI 调用的供应商/模型只在客户端初始化时输出一次。 + +**改动：** + +| 文件 | 改动内容 | +|------|---------| +| `scripts/pipeline.sh` | 每阶段 banner（`━━━ M4 翻译+事件抽取（AI 大模型: deepseek / deepseek-v4-flash）━━━`）；`ai_model_info()` 从 system.yaml 读取场景模型（translation/daily_report/embedding） | +| `scripts/domestic_full.sh` | M1 / 管道阶段标题统一 banner 风格 | +| `llm/pipeline.py` | 创建客户端后显性输出 `AI 大模型（场景 translation）: provider/model` | +| `embedding/pipeline.py` | 向量化日志补充 provider | +| `scheduler/reporter.py` | daily_report 场景客户端初始化时输出 provider/model | + +**测试：** 本地模拟 banner 全部正确（M4/M5/日报 3 处 AI 标注）；pi5 实测客户端初始化日志含 provider/model（deepseek/deepseek-v4-flash、dashscope/text-embedding-v3）；全量 184 passed。 + +--- + ## 2026-08-12 会话成果（三）：全流程终端进度输出 **背景：** `domestic_full.sh` / `pipeline.sh` 原用 `tail -5/-10` 截断输出，终端看不到中间进度。 diff --git a/embedding/pipeline.py b/embedding/pipeline.py index b944a6b..fee89b2 100644 --- a/embedding/pipeline.py +++ b/embedding/pipeline.py @@ -100,8 +100,8 @@ def embed_all_events( # 批量嵌入（按 batch_size 分块，每批输出进度） batch_size = config.batch_size total = len(articles) - logger.info("开始向量化 %d 篇文章（batch_size=%d, model=%s）", - total, batch_size, config.model) + logger.info("开始向量化 %d 篇文章（batch_size=%d, provider=%s, model=%s）", + total, batch_size, config.provider, config.model) for batch_start in range(0, total, batch_size): batch_end = min(batch_start + batch_size, total) diff --git a/llm/pipeline.py b/llm/pipeline.py index 4517465..20231bc 100644 --- a/llm/pipeline.py +++ b/llm/pipeline.py @@ -122,6 +122,10 @@ def translate_all_deduped( # 初始化 LLM 客户端（translation 场景配置见 system.yaml llm_scenes.translation） config = load_llm_config(provider=provider, model=model, scene="translation") client = make_sync_client(config) + logger.info( + "AI 大模型（场景 translation）: provider=%s model=%s", + config.provider, config.model, + ) template = PromptTemplate() # 输出目录 diff --git a/scheduler/reporter.py b/scheduler/reporter.py index bf87ca8..2fc9344 100644 --- a/scheduler/reporter.py +++ b/scheduler/reporter.py @@ -322,6 +322,9 @@ def _call_llm_simple( from llm.client import load_llm_config, make_sync_client _llm_client_cache["config"] = load_llm_config(scene="daily_report") _llm_client_cache[cache_key] = make_sync_client(_llm_client_cache["config"]) + logger.info("AI 大模型（场景 daily_report）: provider=%s model=%s", + _llm_client_cache["config"].provider, + _llm_client_cache["config"].model) config = _llm_client_cache["config"] client = _llm_client_cache[cache_key] diff --git a/scripts/domestic_full.sh b/scripts/domestic_full.sh index 7da0d05..5b05271 100755 --- a/scripts/domestic_full.sh +++ b/scripts/domestic_full.sh @@ -46,14 +46,14 @@ export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true) # 部分源抓取失败不阻塞管道（原语义）；抓取本身按 URL 去重（index.jsonl）， # 已抓取过的 URL 不会重复写入；输出实时显示每源进度 if step_should_run M1_crawl; then - LOG "[1/2] Pi M1 抓取（8G headful + HTTP 代理）..." + LOG "━━━ [1/2] M1 抓取（headful Playwright + HTTP 代理）━━━" bash "$SCRIPT_DIR/domestic_crawl_8g.sh" 2>&1 | tee -a "$FULL_LOG" \ || LOG "WARNING: 部分源抓取失败，继续管道" step_mark M1_crawl fi # ── 2. M2→M6 管道（含日报）── -LOG "[2/2] 全链路管道..." +LOG "━━━ [2/2] M2→M6 管道 + 日报 ━━━" if [ "$RESUME" = "1" ]; then bash "$SCRIPT_DIR/pipeline.sh" --resume 2>&1 | tee -a "$FULL_LOG" else diff --git a/scripts/pipeline.sh b/scripts/pipeline.sh index e8c0d83..81e45f5 100755 --- a/scripts/pipeline.sh +++ b/scripts/pipeline.sh @@ -43,10 +43,33 @@ STEP_LOG="$LOG_DIR/pipeline_$(date +%Y%m%d_%H%M%S).log" LOG "══════ 全链路管道开始（resume=${RESUME}）═══════" LOG "步骤日志: ${STEP_LOG}（终端实时显示完整进度）" +# ── AI 模型信息读取（供阶段 banner 显性展示供应商/模型）── +ai_model_info() { + # $1: 场景名（translation / daily_report）或 "embedding" + # 输出格式: "供应商 / 模型" + .venv/bin/python3 -c " +import sys, yaml +scene = sys.argv[1] +raw = yaml.safe_load(open('configs/system.yaml', encoding='utf-8')) +if scene == 'embedding': + cfg = raw.get('embedding', {}) + p = cfg.get('provider', '?') + m = cfg.get('dashscope_model', '?') +else: + base = raw.get('llm', {}) + sc = (raw.get('llm_scenes', {}) or {}).get(scene, {}) + merged = {**base, **sc} + p = merged.get('provider', '?') + m = merged.get('model') or merged.get(p + '_model', '?') +print(f'{p} / {m}') +" "$1" +} + # 加载 .env export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true) # ── M2: 正文提取 ── +LOG "━━━ M2 正文提取 ━━━" step_run M2_extract .venv/bin/python3 -c " from extractor.pipeline import process_all_sources stats = process_all_sources() @@ -54,20 +77,23 @@ print(f'M2: {stats[\"total_articles\"]} 篇, {stats[\"elapsed_sec\"]:.0f}s') " # ── M3: 去重 ── +LOG "━━━ M3 三层去重 ━━━" step_run M3_dedup .venv/bin/python3 -c " from dedup.pipeline import dedup_all_sources stats = dedup_all_sources() print(f'M3: 唯一 {stats[\"unique\"]}/重复 {stats[\"duplicate\"]}, {stats[\"elapsed_sec\"]:.0f}s') " -# ── M4: 翻译+事件 ── +# ── M4: 翻译+事件（AI 大模型）── +LOG "━━━ M4 翻译+事件抽取（AI 大模型: $(ai_model_info translation)）━━━" step_run M4_translate .venv/bin/python3 -c " from llm.pipeline import translate_all_deduped stats = translate_all_deduped() print(f'M4: {stats[\"success\"]}/{stats[\"total\"]} 篇, {stats[\"elapsed_sec\"]:.0f}s') " -# ── M5: 向量生成 ── +# ── M5: 向量生成（AI 大模型）── +LOG "━━━ M5 向量生成（AI 大模型: $(ai_model_info embedding)）━━━" step_run M5_embed .venv/bin/python3 -c " from embedding.pipeline import embed_all_events stats = embed_all_events() @@ -75,13 +101,15 @@ print(f'M5: {stats[\"success\"]}/{stats[\"total\"]} 篇, {stats[\"elapsed_sec\"] " # ── M6: Qdrant 入库 ── +LOG "━━━ M6 Qdrant 入库 ━━━" step_run M6_index .venv/bin/python3 -c " from vectorstore.pipeline import ingest_all_embeddings stats = ingest_all_embeddings() print(f'M6: {stats[\"ingested\"]}/{stats[\"total\"]} 条, {stats[\"elapsed_sec\"]:.0f}s') " -# ── 日报 ── +# ── 日报（AI 大模型）── +LOG "━━━ 日报生成（AI 大模型: $(ai_model_info daily_report)）━━━" step_run report .venv/bin/python3 -c " from scheduler.reporter import generate_report report_id = generate_report()