feat(db): SQLite 全量迁移至 MySQL(config.yaml 配置化 + 迁移脚本 + 一致性校验)
- config.yaml database.mysql:host/port/db/user/charset 明文可提交;密码经 password_env 引用 .env 的 MYSQL_PASSWORD(AGENT.md §33 密钥不进 git) - config.py _build_mysql_url:URL 优先级 DATABASE_URL env > database.mysql 段 > sqlite 兜底 - pyproject 引入 pymysql>=1.1 - tests/conftest.py 强制每进程 /tmp SQLite(测试绝不触 MySQL 开发库);test_config 覆盖 mysql 组装/密码可选/sqlite 兜底分支 - scripts/migrate_sqlite_to_mysql.py:sqlite→mysql 一次性迁移工具(keyset 分页 + chunk 多值 INSERT + 攒批 commit + 幂等续传 + 低配 MySQL 节流 --throttle-sec + --verify-only 行数与抽样一致性校验);已用于 data/quant.db 约 1605 万行迁移并经校验一致
This commit is contained in:
@@ -10,6 +10,7 @@ import os
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import yaml
|
||||
|
||||
@@ -99,6 +100,26 @@ def _normalize_sqlite_url(url: str) -> str:
|
||||
return f"{_SQLITE_PREFIX}{(PROJECT_ROOT / rest).resolve()}"
|
||||
|
||||
|
||||
def _build_mysql_url(mysql: dict | None) -> str | None:
|
||||
"""由 config.yaml database.mysql 段组装 mysql+pymysql URL。
|
||||
|
||||
规则:仅当 enabled 且 host/user/db 齐全时返回 URL;密码经 password_env
|
||||
指定的环境变量读取(AGENT.md §33 密钥只放 .env),未设置则按空密码处理。
|
||||
"""
|
||||
if not mysql or not mysql.get("enabled"):
|
||||
return None
|
||||
host = mysql.get("host")
|
||||
user = mysql.get("user")
|
||||
db = mysql.get("db")
|
||||
if not (host and user and db):
|
||||
return None
|
||||
port = mysql.get("port") or 3306
|
||||
charset = mysql.get("charset") or "utf8mb4"
|
||||
password = os.environ.get(mysql.get("password_env") or "MYSQL_PASSWORD", "")
|
||||
auth = f"{quote_plus(user)}:{quote_plus(password)}" if password else quote_plus(user)
|
||||
return f"mysql+pymysql://{auth}@{host}:{port}/{db}?charset={charset}"
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
"""加载并缓存 Settings。config / env 路径可通过参数覆盖以便测试。"""
|
||||
@@ -115,7 +136,10 @@ def get_settings() -> Settings:
|
||||
llm_model_env = agent_llm.get("model_env") or "LLM_MODEL"
|
||||
|
||||
default_url = "sqlite:///./data/quant.db"
|
||||
database_url = os.environ.get(url_env) or default_url
|
||||
# URL 优先级:环境变量(DATABASE_URL) > config.yaml database.mysql 段 > SQLite 兜底
|
||||
database_url = os.environ.get(url_env) or _build_mysql_url(
|
||||
_deep(cfg, "database.mysql") or {}
|
||||
) or default_url
|
||||
|
||||
migrations_rel = _deep(cfg, "database.migrations_dir") or (
|
||||
"app/infrastructure/persistence/migrations"
|
||||
|
||||
Reference in New Issue
Block a user