perf(backend): 内存优化三项——全市场研究不再占满 8G
1) 数据装配流式+列裁剪:Repository 新增 stream_range_many_columns(只 SELECT 所需列、SQL 侧转 REAL、yield_per 分批),引擎按 required_columns 取数 (LocalEngine 仅 close+因子字段),消除 ORM/Decimal 全量物化; 2) 研究 Job 独立子进程执行(job.mode=subprocess):python -m app.cli.run_job 在子进程内设 RLIMIT_AS 上限,OOM 归档 failed 而非拖垮 API worker; 子进程异常退出由父进程补记 failed;并发上限 2; 3) 服务启动清理:残留 queued/running Job 标记 failed(防永久 running)。 实测同款全市场回测:uvicorn worker RSS 稳定 ~220MB,任务峰值内存由 4.1GB+ 降至 ~470MB,24s 完成并归档(此前 43s 未完成即 OOM)。 新增/更新测试 96 passed,ruff 干净。
This commit is contained in:
@@ -72,6 +72,9 @@ class Settings:
|
||||
llm_base_url: str | None
|
||||
llm_model: str
|
||||
storage: dict[str, Path]
|
||||
job_mode: str
|
||||
job_memory_limit_gb: int
|
||||
job_max_concurrent: int
|
||||
config_path: Path = CONFIG_PATH
|
||||
env_path: Path = ENV_PATH
|
||||
project_root: Path = PROJECT_ROOT
|
||||
@@ -118,6 +121,19 @@ def get_settings() -> Settings:
|
||||
"app/infrastructure/persistence/migrations"
|
||||
)
|
||||
|
||||
# Job 执行:环境变量 JOB_MODE / QLIB_JOB_MEM_LIMIT_GB 可覆盖 config.yaml(测试用)
|
||||
job_cfg = _deep(cfg, "job") or {}
|
||||
try:
|
||||
job_memory_limit_gb = int(
|
||||
os.environ.get("QLIB_JOB_MEM_LIMIT_GB") or job_cfg.get("max_memory_gb") or 6
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
job_memory_limit_gb = 6
|
||||
try:
|
||||
job_max_concurrent = int(os.environ.get("QLIB_JOB_MAX_CONCURRENT") or job_cfg.get("max_concurrent_jobs") or 2)
|
||||
except (TypeError, ValueError):
|
||||
job_max_concurrent = 2
|
||||
|
||||
return Settings(
|
||||
app_name=str(_deep(cfg, "app.name") or "qlib-platform"),
|
||||
app_version=str(_deep(cfg, "app.version") or "0.1.0"),
|
||||
@@ -134,4 +150,7 @@ def get_settings() -> Settings:
|
||||
llm_base_url=os.environ.get(llm_url_env) or (agent_llm.get("base_url") or None),
|
||||
llm_model=os.environ.get(llm_model_env) or agent_llm.get("model") or "qwen-plus",
|
||||
storage=_resolve_storage_dirs(cfg),
|
||||
job_mode=os.environ.get("JOB_MODE") or job_cfg.get("mode") or "subprocess",
|
||||
job_memory_limit_gb=job_memory_limit_gb,
|
||||
job_max_concurrent=job_max_concurrent,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user