simon
60f8c263f2
从 CCTV 主页抓取《新闻联播》，下载 → 转 MP3/WAV → 静音切分 → ASR 识别 → LLM 校对 → 切分为单条新闻 → 入库 MySQL。 主要内容： - 全链路：getVideo5 抓取下载、audioRead 转写、deepseek 校对与切分、newsProcess 入库 - 可移植化：配置分层，.env 只放密钥、config.yml 放模型/接入点/路由/参数 - 可换供应商：endpoints（kind/base_url/api_key_env/extra_body）+ routes 按环节选路 - 数据保真：数值事实守卫，校对改动数字/年份/届次则整片回退 ASR 原文； 识别不完整不发布该日精编，避免半天内容被当成完整一天 - 定时任务：systemd 每天 21:00，失败 21:30 / 22:00 重试； 只缺切分时只重跑切分（省掉全部 ASR），用 state/.asr_complete_* 标记判定阶段 - 隧道自愈：13306 不通时自动执行 autossh.sh（所有入口共用，systemd 托管时只等待） - 中间产物每日清理；97 项离线自检（配置/清理/事实守卫/解析/隧道）
240 lines
10 KiB
Python
240 lines
10 KiB
Python
"""MySQLDB —— 自包含的数据库访问层
|
||
|
||
背景：原实现位于父项目 `djapi/api/utils/mysql_handler.py`，本项目独立运行后该模块缺失
|
||
（`mysqlHandle.py` 通过 sys.path 引用父目录，换机即 ModuleNotFoundError）。
|
||
本文件把该实现内置回项目，**保持原有 4 个接口不变**，只修掉原实现的缺陷：
|
||
|
||
db = MySQLDB() # 无参：读环境变量
|
||
rows = db.query_data(table, columns, where, params) # -> list[dict]
|
||
new_id = db.insert_data(table, {col: val}) # -> lastrowid
|
||
db.update_data(table, {col: val}, where, params) # -> rowcount
|
||
db.close()
|
||
|
||
修复点（对照 docs/BUGS.md）：
|
||
- B10-1 连接失败时原实现只 print 然后留下 connection=None，后续再抛
|
||
`AttributeError: 'NoneType' object has no attribute 'cursor'`；现改为**立即抛错**
|
||
并给出目标地址，便于定位配置问题。
|
||
- B10-2 原实现 `finally: if cursor:` 在 `cursor = self.connection.cursor()` 之前抛错时
|
||
会触发 `local variable 'cursor' referenced before assignment`（生产日志出现过）；
|
||
现把 cursor 提前初始化为 None。
|
||
- 新增 `execute()` / `insert_many()`，用于按日期替换（幂等写库，见 B5）与批量事务插入。
|
||
|
||
配置来源（2026-09-25 重构）：
|
||
- 地址/端口/用户/库名 ← config.yml 的 `mysql.*`
|
||
- 口令 ← .env 的 `MYSQL_PASSWORD`
|
||
- 以上均可被构造参数直接覆盖（保持与原实现一致的用法）
|
||
- 隧道自愈（2026-09-25）：连接失败时调用 `tunnel.ensure_tunnel()`，
|
||
端口不通则按 config.yml 的 `mysql.tunnel` 执行 autossh.sh 后重试一次（见 tunnel.py）
|
||
"""
|
||
import logging
|
||
|
||
import config
|
||
import env # noqa: F401 — 保证 .env（敏感项）已加载
|
||
import mysql.connector
|
||
import tunnel # 隧道自愈：端口不通时自动执行 autossh.sh
|
||
from mysql.connector import Error
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class MySQLDB:
|
||
"""轻量 MySQL/MariaDB 访问封装（接口与原 utils.mysql_handler.MySQLDB 兼容）"""
|
||
|
||
def __init__(self, host=None, port=None, username=None, password=None, database=None,
|
||
connect_timeout=10):
|
||
cfg = config.db_config()
|
||
self.host = host or cfg['host']
|
||
self.port = int(port or cfg['port'])
|
||
self.username = username or cfg['username']
|
||
self.password = password if password is not None else cfg['password']
|
||
self.database = database or cfg['database']
|
||
self.connect_timeout = connect_timeout
|
||
self.connection = None
|
||
self.connect()
|
||
|
||
# ------------------------------------------------------------------ 连接
|
||
@property
|
||
def target(self):
|
||
return f"{self.username}@{self.host}:{self.port}/{self.database}"
|
||
|
||
def connect(self):
|
||
"""建立连接；失败时先尝试隧道自愈，仍失败才抛异常（不再静默吞掉）"""
|
||
last_error = None
|
||
for attempt in (1, 2):
|
||
try:
|
||
self.connection = mysql.connector.connect(
|
||
host=self.host,
|
||
port=self.port,
|
||
user=self.username,
|
||
password=self.password,
|
||
database=self.database,
|
||
connection_timeout=self.connect_timeout,
|
||
charset='utf8mb4',
|
||
)
|
||
if attempt == 2:
|
||
logger.info("✓ 隧道自愈后重连成功（%s）", self.target)
|
||
logger.debug("已连接数据库 %s", self.target)
|
||
return self.connection
|
||
except Error as e:
|
||
self.connection = None
|
||
last_error = e
|
||
if attempt == 2:
|
||
break
|
||
# 只在失败时介入：正常路径不增加任何探测开销。
|
||
# 端口不通就按 config.yml 的 mysql.tunnel 执行 autossh.sh（见 tunnel.py）
|
||
healed = tunnel.ensure_tunnel(self.host, self.port, reason=f"MySQL 连接失败: {e}")
|
||
if not healed.get('healed') and healed['status'] not in (
|
||
tunnel.ST_ALREADY_OPEN, tunnel.ST_SYSTEMD_WAITED, tunnel.ST_SCRIPT_STARTED):
|
||
break
|
||
raise RuntimeError(
|
||
f"MySQL 连接失败（{self.target}）：{last_error}\n"
|
||
"请检查 config.yml 的 mysql.host/port/user/database 与 .env 的 MYSQL_PASSWORD；"
|
||
f"若经 SSH 隧道访问，已尝试执行 {config.get('mysql.tunnel.script', 'autossh.sh')}"
|
||
"（可手动运行 `python tunnel.py` 查看隧道状态）"
|
||
) from last_error
|
||
|
||
def _ensure_connection(self):
|
||
"""连接可用性检查（长流程中连接可能被服务端断开）"""
|
||
if self.connection is None:
|
||
self.connect()
|
||
return self.connection
|
||
try:
|
||
self.connection.ping(reconnect=True, attempts=3, delay=1)
|
||
except Error:
|
||
self.connect()
|
||
return self.connection
|
||
|
||
# ------------------------------------------------------------------ 写
|
||
def insert_data(self, table, data):
|
||
"""插入一行，返回自增主键（失败返回 None）"""
|
||
cursor = None
|
||
try:
|
||
conn = self._ensure_connection()
|
||
cursor = conn.cursor()
|
||
columns = ', '.join(data.keys())
|
||
placeholders = ', '.join(['%s'] * len(data))
|
||
query = f"INSERT INTO {table} ({columns}) VALUES ({placeholders})"
|
||
cursor.execute(query, tuple(data.values()))
|
||
conn.commit()
|
||
logger.debug("插入 %s 成功，影响行数 %s", table, cursor.rowcount)
|
||
return cursor.lastrowid
|
||
except Error as e:
|
||
logger.error("插入 %s 失败: %s", table, e)
|
||
return None
|
||
finally:
|
||
if cursor is not None:
|
||
cursor.close()
|
||
|
||
def insert_many(self, table, rows, chunk_size=500):
|
||
"""批量插入（同一事务提交一次），rows 为 dict 列表；返回成功写入的行数"""
|
||
if not rows:
|
||
return 0
|
||
cursor = None
|
||
try:
|
||
conn = self._ensure_connection()
|
||
cursor = conn.cursor()
|
||
columns = list(rows[0].keys())
|
||
col_sql = ', '.join(columns)
|
||
placeholders = ', '.join(['%s'] * len(columns))
|
||
query = f"INSERT INTO {table} ({col_sql}) VALUES ({placeholders})"
|
||
values = [tuple(r[c] for c in columns) for r in rows]
|
||
written = 0
|
||
for start in range(0, len(values), chunk_size):
|
||
batch = values[start:start + chunk_size]
|
||
cursor.executemany(query, batch)
|
||
written += cursor.rowcount
|
||
conn.commit()
|
||
logger.debug("批量插入 %s 成功，%d 行", table, written)
|
||
return written
|
||
except Error as e:
|
||
if self.connection is not None:
|
||
self.connection.rollback()
|
||
logger.error("批量插入 %s 失败，已回滚: %s", table, e)
|
||
return 0
|
||
finally:
|
||
if cursor is not None:
|
||
cursor.close()
|
||
|
||
def update_data(self, table, data, where, params=None):
|
||
"""按条件更新，返回受影响行数"""
|
||
cursor = None
|
||
try:
|
||
conn = self._ensure_connection()
|
||
cursor = conn.cursor()
|
||
set_clause = ', '.join([f"{key} = %s" for key in data.keys()])
|
||
query = f"UPDATE {table} SET {set_clause} WHERE {where}"
|
||
all_params = tuple(data.values()) + tuple(params or ())
|
||
cursor.execute(query, all_params)
|
||
conn.commit()
|
||
logger.debug("更新 %s 成功，影响行数 %s", table, cursor.rowcount)
|
||
return cursor.rowcount
|
||
except Error as e:
|
||
logger.error("更新 %s 失败: %s", table, e)
|
||
return 0
|
||
finally:
|
||
if cursor is not None:
|
||
cursor.close()
|
||
|
||
def execute(self, sql, params=None, commit=True):
|
||
"""执行任意写 SQL，返回受影响行数（用于 DELETE / DDL 等）"""
|
||
cursor = None
|
||
try:
|
||
conn = self._ensure_connection()
|
||
cursor = conn.cursor()
|
||
cursor.execute(sql, tuple(params or ()))
|
||
if commit:
|
||
conn.commit()
|
||
return cursor.rowcount
|
||
except Error as e:
|
||
if commit and self.connection is not None:
|
||
self.connection.rollback()
|
||
logger.error("执行失败: %s | SQL=%s", e, sql.split('\n')[0][:120])
|
||
return 0
|
||
finally:
|
||
if cursor is not None:
|
||
cursor.close()
|
||
|
||
# ------------------------------------------------------------------ 读
|
||
def query_data(self, table, columns="*", where=None, params=None):
|
||
"""查询，返回 list[dict]（失败返回 []）"""
|
||
cursor = None
|
||
try:
|
||
conn = self._ensure_connection()
|
||
cursor = conn.cursor(dictionary=True)
|
||
query = f"SELECT {columns} FROM {table}"
|
||
if where:
|
||
query += f" WHERE {where}"
|
||
logger.debug("查询: %s", query)
|
||
cursor.execute(query, tuple(params or ()))
|
||
return cursor.fetchall()
|
||
except Error as e:
|
||
logger.error("查询 %s 失败: %s", table, e)
|
||
return []
|
||
finally:
|
||
if cursor is not None:
|
||
cursor.close()
|
||
|
||
def query_one(self, table, columns="*", where=None, params=None):
|
||
"""查询单行，无结果返回 None"""
|
||
rows = self.query_data(table, columns, where, params)
|
||
return rows[0] if rows else None
|
||
|
||
# ------------------------------------------------------------------ 收尾
|
||
def close(self):
|
||
"""关闭连接（可重复调用）"""
|
||
if self.connection is not None:
|
||
try:
|
||
if self.connection.is_connected():
|
||
self.connection.close()
|
||
logger.debug("数据库连接已关闭")
|
||
except Error as e:
|
||
logger.warning("关闭数据库连接出错: %s", e)
|
||
finally:
|
||
self.connection = None
|
||
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def __exit__(self, exc_type, exc, tb):
|
||
self.close()
|
||
return False |