simon/qlib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(db): SQLite 全量迁移至 MySQL（config.yaml 配置化 + 迁移脚本 + 一致性校验）

Browse Source 
- config.yaml database.mysql：host/port/db/user/charset 明文可提交；密码经 password_env
  引用 .env 的 MYSQL_PASSWORD（AGENT.md §33 密钥不进 git）
- config.py _build_mysql_url：URL 优先级 DATABASE_URL env > database.mysql 段 > sqlite 兜底
- pyproject 引入 pymysql>=1.1
- tests/conftest.py 强制每进程 /tmp SQLite（测试绝不触 MySQL 开发库）；test_config 覆盖
  mysql 组装/密码可选/sqlite 兜底分支
- scripts/migrate_sqlite_to_mysql.py：sqlite→mysql 一次性迁移工具（keyset 分页 + chunk
  多值 INSERT + 攒批 commit + 幂等续传 + 低配 MySQL 节流 --throttle-sec + --verify-only
  行数与抽样一致性校验）；已用于 data/quant.db 约 1605 万行迁移并经校验一致
This commit is contained in:
Simon
2026-09-08 23:58:48 +08:00
parent a3fabf9ae9
commit 6c2f198261
8 changed files with 409 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/conftest.py
+7 -1
View File
@@ -1,8 +1,10 @@
"""全局测试配置。
- 强制 SQLite 临时文件数据库：默认库已是 MySQL（config.yaml database.mysql），
测试绝不允许触碰开发 MySQL（qlib@192.168.1.10）；该 env 在 app 模块
首次导入前生效，SessionLocal/engine 会按它构造。
- JOB_MODE=local：单测/CI 里 Job 在本进程执行（不 spawn 子进程、不依赖安装态）
- QLIB_SKIP_STALE_JOB_CLEANUP=1：TestClient 启动 lifespan 不清理残留 Job
（避免连接/写入开发库 data/quant.db）
必须在任何 app / config 模块首次导入前生效，故放在 conftest 模块顶层。
"""
@@ -11,6 +13,10 @@ from __future__ import annotations
import os
# 测试数据库：每进程唯一 /tmp sqlite 文件（隔离、不落项目目录、不触 MySQL）
_TEST_DB = f"/tmp/qlib-pytest-{os.getpid()}.db"
os.environ["DATABASE_URL"] = f"sqlite:///{_TEST_DB}"
# 强制（不是 setdefault）：测试绝不 spawn 研究子进程 / 不触碰开发库
os.environ["JOB_MODE"] = "local"
os.environ["QLIB_SKIP_STALE_JOB_CLEANUP"] = "1"
backend/tests/test_config.py
+63 -14
View File
@@ -1,9 +1,10 @@
"""配置层测试：默认 SQLite 相对路径解析、config.yaml / .env 加载约定。"""
"""配置层测试：默认 MySQL URL 组装（config.yaml database.mysql）、.env 加载约定、
SQLite 兜底逻辑。conftest 已强制 DATABASE_URL=tmp sqlite，本文件内用
monkeypatch/cache_clear 单独验证「无 env 时走 mysql 段」的分支（只组装不连接）。"""
from __future__ import annotations
import os
from pathlib import Path
from app.core.config import PROJECT_ROOT, _load_env_file, get_settings
@@ -14,14 +15,62 @@ def test_project_root_points_to_repo_root() -> None:
assert (PROJECT_ROOT / "backend").is_dir()
def test_default_database_url_resolves_to_project_data_dir() -> None:
def test_test_runner_isolation_uses_tmp_sqlite() -> None:
"""conftest 强制每进程唯一 /tmp sqlite，测试绝不触碰 MySQL 开发库。"""
settings = get_settings()
assert settings.database_url.startswith("sqlite:///")
# 相对路径应解析到 <项目根>/data/quant.db
assert settings.database_url.endswith("/data/quant.db")
db_path = Path(settings.database_url.removeprefix("sqlite:///"))
assert db_path.is_absolute()
assert db_path == PROJECT_ROOT / "data" / "quant.db"
assert settings.database_url.startswith("sqlite:////tmp/qlib-pytest-")
def test_mysql_default_from_config_yaml(monkeypatch) -> None:
"""未设 DATABASE_URL 时，config.yaml database.mysql 段组装 MySQL URL（不连接）。"""
monkeypatch.delenv("DATABASE_URL", raising=False)
get_settings.cache_clear()
try:
url = get_settings().database_url
finally:
get_settings.cache_clear()
assert url.startswith("mysql+pymysql://qlib:")
assert "192.168.1.10:3306/qlib?charset=utf8mb4" in url
def test_build_mysql_url(monkeypatch) -> None:
from app.core.config import _build_mysql_url
monkeypatch.setenv("MYSQL_PASSWORD", "p@ss:word")
cfg = {
"enabled": True,
"host": "10.0.0.2",
"port": 3307,
"db": "q",
"user": "u",
"password_env": "MYSQL_PASSWORD",
}
url = _build_mysql_url(cfg)
assert url == "mysql+pymysql://u:p%40ss%3Aword@10.0.0.2:3307/q?charset=utf8mb4"
# 未启用 / 缺字段 → None（回退 sqlite 兜底）
assert _build_mysql_url({**cfg, "enabled": False}) is None
assert _build_mysql_url({**cfg, "host": None}) is None
assert _build_mysql_url(None) is None
def test_mysql_url_password_optional(monkeypatch) -> None:
"""密码留空（未设 env）也可组装 URL —— 方便仅内网/免密场景。"""
from app.core.config import _build_mysql_url
monkeypatch.delenv("MYSQL_PASSWORD", raising=False)
cfg = {"enabled": True, "host": "h", "db": "d", "user": "u"}
assert _build_mysql_url(cfg) == "mysql+pymysql://u@h:3306/d?charset=utf8mb4"
def test_sqlite_url_normalization_and_fallback() -> None:
"""仅 sqlite 相对路径被解析为项目根绝对路径；其它 URL 原样透传。"""
from app.core.config import _normalize_sqlite_url
assert _normalize_sqlite_url("mysql+pymysql://u:p@h/d") == "mysql+pymysql://u:p@h/d"
abs_url = _normalize_sqlite_url("sqlite:///./data/quant.db")
assert abs_url.startswith("sqlite:///")
# 绝对路径 sqlite 不再重复解析
assert _normalize_sqlite_url("sqlite:////abs/x.db") == "sqlite:////abs/x.db"
def test_settings_loaded_from_config_yaml() -> None:
@@ -32,7 +81,7 @@ def test_settings_loaded_from_config_yaml() -> None:
assert settings.data_source_fallback == "sina"
def test_env_file_loading(tmp_path: Path, monkeypatch) -> None:
def test_env_file_loading(tmp_path, monkeypatch) -> None:
monkeypatch.delenv("TUSHARE_TOKEN", raising=False)
env_file = tmp_path / ".env"
env_file.write_text(
@@ -57,14 +106,14 @@ def test_llm_from_config_yaml() -> None:
def test_llm_env_overrides_yaml(monkeypatch) -> None:
from app.core.config import get_settings
monkeypatch.setenv("LLM_MODEL", "env-model")
monkeypatch.setenv("LLM_BASE_URL", "https://example.com/v1")
monkeypatch.setenv("LLM_API_KEY", "sk-test")
get_settings.cache_clear() # Settings 有 lru_cache，刷新以读取新环境变量
s = get_settings()
try:
s = get_settings()
finally:
get_settings.cache_clear()
assert s.llm_model == "env-model"
assert s.llm_base_url == "https://example.com/v1"
assert s.llm_api_key == "sk-test"
get_settings.cache_clear()