《新闻联播》每日抓取入库：全链路 + 可移植化 + 定时任务

从 CCTV 主页抓取《新闻联播》，下载 → 转 MP3/WAV → 静音切分 → ASR 识别 → LLM 校对 → 切分为单条新闻 → 入库 MySQL。主要内容： - 全链路：getVideo5 抓取下载、audioRead 转写、deepseek 校对与切分、newsProcess 入库 - 可移植化：配置分层，.env 只放密钥、config.yml 放模型/接入点/路由/参数 - 可换供应商：endpoints（kind/base_url/api_key_env/extra_body）+ routes 按环节选路 - 数据保真：数值事实守卫，校对改动数字/年份/届次则整片回退 ASR 原文；识别不完整不发布该日精编，避免半天内容被当成完整一天 - 定时任务：systemd 每天 21:00，失败 21:30 / 22:00 重试；只缺切分时只重跑切分（省掉全部 ASR），用 state/.asr_complete_* 标记判定阶段 - 隧道自愈：13306 不通时自动执行 autossh.sh（所有入口共用，systemd 托管时只等待） - 中间产物每日清理；97 项离线自检（配置/清理/事实守卫/解析/隧道）
2026-09-25 11:17:46 +08:00
commit 60f8c263f2
35 changed files with 5770 additions and 0 deletions
@@ -0,0 +1,240 @@
+"""MySQLDB —— 自包含的数据库访问层
+
+背景：原实现位于父项目 `djapi/api/utils/mysql_handler.py`，本项目独立运行后该模块缺失
+（`mysqlHandle.py` 通过 sys.path 引用父目录，换机即 ModuleNotFoundError）。
+本文件把该实现内置回项目，**保持原有 4 个接口不变**，只修掉原实现的缺陷：
+
+    db = MySQLDB()                                                    # 无参：读环境变量
+    rows = db.query_data(table, columns, where, params)               # -> list[dict]
+    new_id = db.insert_data(table, {col: val})                        # -> lastrowid
+    db.update_data(table, {col: val}, where, params)                  # -> rowcount
+    db.close()
+
+修复点（对照 docs/BUGS.md）：
+- B10-1 连接失败时原实现只 print 然后留下 connection=None，后续再抛
+  `AttributeError: 'NoneType' object has no attribute 'cursor'`；现改为**立即抛错**
+  并给出目标地址，便于定位配置问题。
+- B10-2 原实现 `finally: if cursor:` 在 `cursor = self.connection.cursor()` 之前抛错时
+  会触发 `local variable 'cursor' referenced before assignment`（生产日志出现过）；
+  现把 cursor 提前初始化为 None。
+- 新增 `execute()` / `insert_many()`，用于按日期替换（幂等写库，见 B5）与批量事务插入。
+
+配置来源（2026-09-25 重构）：
+- 地址/端口/用户/库名  ← config.yml 的 `mysql.*`
+- 口令                ← .env 的 `MYSQL_PASSWORD`
+- 以上均可被构造参数直接覆盖（保持与原实现一致的用法）
+- 隧道自愈（2026-09-25）：连接失败时调用 `tunnel.ensure_tunnel()`，
+  端口不通则按 config.yml 的 `mysql.tunnel` 执行 autossh.sh 后重试一次（见 tunnel.py）
+"""
+import logging
+
+import config
+import env  # noqa: F401 — 保证 .env（敏感项）已加载
+import mysql.connector
+import tunnel  # 隧道自愈：端口不通时自动执行 autossh.sh
+from mysql.connector import Error
+
+logger = logging.getLogger(__name__)
+
+
+class MySQLDB:
+    """轻量 MySQL/MariaDB 访问封装（接口与原 utils.mysql_handler.MySQLDB 兼容）"""
+
+    def __init__(self, host=None, port=None, username=None, password=None, database=None,
+                 connect_timeout=10):
+        cfg = config.db_config()
+        self.host = host or cfg['host']
+        self.port = int(port or cfg['port'])
+        self.username = username or cfg['username']
+        self.password = password if password is not None else cfg['password']
+        self.database = database or cfg['database']
+        self.connect_timeout = connect_timeout
+        self.connection = None
+        self.connect()
+
+    # ------------------------------------------------------------------ 连接
+    @property
+    def target(self):
+        return f"{self.username}@{self.host}:{self.port}/{self.database}"
+
+    def connect(self):
+        """建立连接；失败时先尝试隧道自愈，仍失败才抛异常（不再静默吞掉）"""
+        last_error = None
+        for attempt in (1, 2):
+            try:
+                self.connection = mysql.connector.connect(
+                    host=self.host,
+                    port=self.port,
+                    user=self.username,
+                    password=self.password,
+                    database=self.database,
+                    connection_timeout=self.connect_timeout,
+                    charset='utf8mb4',
+                )
+                if attempt == 2:
+                    logger.info("✓ 隧道自愈后重连成功（%s）", self.target)
+                logger.debug("已连接数据库 %s", self.target)
+                return self.connection
+            except Error as e:
+                self.connection = None
+                last_error = e
+                if attempt == 2:
+                    break
+                # 只在失败时介入：正常路径不增加任何探测开销。
+                # 端口不通就按 config.yml 的 mysql.tunnel 执行 autossh.sh（见 tunnel.py）
+                healed = tunnel.ensure_tunnel(self.host, self.port, reason=f"MySQL 连接失败: {e}")
+                if not healed.get('healed') and healed['status'] not in (
+                        tunnel.ST_ALREADY_OPEN, tunnel.ST_SYSTEMD_WAITED, tunnel.ST_SCRIPT_STARTED):
+                    break
+        raise RuntimeError(
+            f"MySQL 连接失败（{self.target}）：{last_error}\n"
+            "请检查 config.yml 的 mysql.host/port/user/database 与 .env 的 MYSQL_PASSWORD；"
+            f"若经 SSH 隧道访问，已尝试执行 {config.get('mysql.tunnel.script', 'autossh.sh')}"
+            "（可手动运行 `python tunnel.py` 查看隧道状态）"
+        ) from last_error
+
+    def _ensure_connection(self):
+        """连接可用性检查（长流程中连接可能被服务端断开）"""
+        if self.connection is None:
+            self.connect()
+            return self.connection
+        try:
+            self.connection.ping(reconnect=True, attempts=3, delay=1)
+        except Error:
+            self.connect()
+        return self.connection
+
+    # ------------------------------------------------------------------ 写
+    def insert_data(self, table, data):
+        """插入一行，返回自增主键（失败返回 None）"""
+        cursor = None
+        try:
+            conn = self._ensure_connection()
+            cursor = conn.cursor()
+            columns = ', '.join(data.keys())
+            placeholders = ', '.join(['%s'] * len(data))
+            query = f"INSERT INTO {table} ({columns}) VALUES ({placeholders})"
+            cursor.execute(query, tuple(data.values()))
+            conn.commit()
+            logger.debug("插入 %s 成功，影响行数 %s", table, cursor.rowcount)
+            return cursor.lastrowid
+        except Error as e:
+            logger.error("插入 %s 失败: %s", table, e)
+            return None
+        finally:
+            if cursor is not None:
+                cursor.close()
+
+    def insert_many(self, table, rows, chunk_size=500):
+        """批量插入（同一事务提交一次），rows 为 dict 列表；返回成功写入的行数"""
+        if not rows:
+            return 0
+        cursor = None
+        try:
+            conn = self._ensure_connection()
+            cursor = conn.cursor()
+            columns = list(rows[0].keys())
+            col_sql = ', '.join(columns)
+            placeholders = ', '.join(['%s'] * len(columns))
+            query = f"INSERT INTO {table} ({col_sql}) VALUES ({placeholders})"
+            values = [tuple(r[c] for c in columns) for r in rows]
+            written = 0
+            for start in range(0, len(values), chunk_size):
+                batch = values[start:start + chunk_size]
+                cursor.executemany(query, batch)
+                written += cursor.rowcount
+            conn.commit()
+            logger.debug("批量插入 %s 成功，%d 行", table, written)
+            return written
+        except Error as e:
+            if self.connection is not None:
+                self.connection.rollback()
+            logger.error("批量插入 %s 失败，已回滚: %s", table, e)
+            return 0
+        finally:
+            if cursor is not None:
+                cursor.close()
+
+    def update_data(self, table, data, where, params=None):
+        """按条件更新，返回受影响行数"""
+        cursor = None
+        try:
+            conn = self._ensure_connection()
+            cursor = conn.cursor()
+            set_clause = ', '.join([f"{key} = %s" for key in data.keys()])
+            query = f"UPDATE {table} SET {set_clause} WHERE {where}"
+            all_params = tuple(data.values()) + tuple(params or ())
+            cursor.execute(query, all_params)
+            conn.commit()
+            logger.debug("更新 %s 成功，影响行数 %s", table, cursor.rowcount)
+            return cursor.rowcount
+        except Error as e:
+            logger.error("更新 %s 失败: %s", table, e)
+            return 0
+        finally:
+            if cursor is not None:
+                cursor.close()
+
+    def execute(self, sql, params=None, commit=True):
+        """执行任意写 SQL，返回受影响行数（用于 DELETE / DDL 等）"""
+        cursor = None
+        try:
+            conn = self._ensure_connection()
+            cursor = conn.cursor()
+            cursor.execute(sql, tuple(params or ()))
+            if commit:
+                conn.commit()
+            return cursor.rowcount
+        except Error as e:
+            if commit and self.connection is not None:
+                self.connection.rollback()
+            logger.error("执行失败: %s | SQL=%s", e, sql.split('\n')[0][:120])
+            return 0
+        finally:
+            if cursor is not None:
+                cursor.close()
+
+    # ------------------------------------------------------------------ 读
+    def query_data(self, table, columns="*", where=None, params=None):
+        """查询，返回 list[dict]（失败返回 []）"""
+        cursor = None
+        try:
+            conn = self._ensure_connection()
+            cursor = conn.cursor(dictionary=True)
+            query = f"SELECT {columns} FROM {table}"
+            if where:
+                query += f" WHERE {where}"
+            logger.debug("查询: %s", query)
+            cursor.execute(query, tuple(params or ()))
+            return cursor.fetchall()
+        except Error as e:
+            logger.error("查询 %s 失败: %s", table, e)
+            return []
+        finally:
+            if cursor is not None:
+                cursor.close()
+
+    def query_one(self, table, columns="*", where=None, params=None):
+        """查询单行，无结果返回 None"""
+        rows = self.query_data(table, columns, where, params)
+        return rows[0] if rows else None
+
+    # ------------------------------------------------------------------ 收尾
+    def close(self):
+        """关闭连接（可重复调用）"""
+        if self.connection is not None:
+            try:
+                if self.connection.is_connected():
+                    self.connection.close()
+                    logger.debug("数据库连接已关闭")
+            except Error as e:
+                logger.warning("关闭数据库连接出错: %s", e)
+            finally:
+                self.connection = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        self.close()
+        return False