perf(backend): 内存优化三项——全市场研究不再占满 8G
1) 数据装配流式+列裁剪：Repository 新增 stream_range_many_columns（只 SELECT 所需列、SQL 侧转 REAL、yield_per 分批），引擎按 required_columns 取数 （LocalEngine 仅 close+因子字段），消除 ORM/Decimal 全量物化； 2) 研究 Job 独立子进程执行（job.mode=subprocess）：python -m app.cli.run_job 在子进程内设 RLIMIT_AS 上限，OOM 归档 failed 而非拖垮 API worker； 子进程异常退出由父进程补记 failed；并发上限 2； 3) 服务启动清理：残留 queued/running Job 标记 failed（防永久 running）。 实测同款全市场回测：uvicorn worker RSS 稳定 ~220MB，任务峰值内存由 4.1GB+ 降至 ~470MB，24s 完成并归档（此前 43s 未完成即 OOM）。 新增/更新测试 96 passed，ruff 干净。
This commit is contained in:
Simon
@@ -1,14 +1,23 @@
|
||||
"""Job 执行编排与 Experiment 归档（Phase 4）。
|
||||
"""Job 执行编排与 Experiment 归档（Phase 4）+ 内存隔离执行（内存优化专项）。
|
||||
|
||||
execute_job 在后台运行：状态机 queued→running→(success|failed)，
|
||||
execute_job 运行状态机 queued→running→(success|failed)，
|
||||
成功时自动把 spec/result 存为 Experiment（含代码版本），实现「研究可复现」（AGENT.md §21）。
|
||||
独立 Session 生命周期（不依赖请求 scope），可被 FastAPI BackgroundTasks 或测试直接调用。
|
||||
|
||||
执行模式（config job.mode）：
|
||||
- local —— 本进程内执行（开发 / 单测；无内存隔离）
|
||||
- subprocess —— 独立子进程执行（python -m app.cli.run_job <job_id>），
|
||||
子进程内施加 RLIMIT_AS 上限（config job.max_memory_gb），研究任务 OOM 只会
|
||||
MemoryError 失败归档，不会拖垮 API worker；父进程在子进程异常退出时补记 failed。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
@@ -24,7 +33,13 @@ from app.domain.entities.research import (
|
||||
)
|
||||
from app.quant.service import ResearchService
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[3]
|
||||
# backend/app/application/services/job_executor.py → parents[4] = 项目根
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[4]
|
||||
BACKEND_ROOT = PROJECT_ROOT / "backend"
|
||||
|
||||
# 本进程内正在运行的子进程 Job（并发上限保护宿主内存）
|
||||
_active_jobs: dict[str, subprocess.Popen | None] = {}
|
||||
_active_lock = threading.Lock()
|
||||
|
||||
|
||||
def new_id(prefix: str) -> str:
|
||||
@@ -171,7 +186,10 @@ def default_factories() -> dict:
|
||||
|
||||
|
||||
def submit_and_run(spec: ResearchSpec, *, factories: dict | None = None) -> JobRecord:
|
||||
"""同步创建并执行一个 Job（复用 Job 状态机与 Experiment 归档），返回终态 Job。"""
|
||||
"""创建并执行一个 Job（复用 Job 状态机与 Experiment 归档），返回终态 Job。
|
||||
|
||||
按 job.mode 调度：subprocess 模式在独立进程内执行（内存隔离），否则本进程。
|
||||
"""
|
||||
facts = factories or default_factories()
|
||||
session_factory = facts["session_factory"]
|
||||
job_repo = facts["job_repo_factory"]
|
||||
@@ -185,16 +203,178 @@ def submit_and_run(spec: ResearchSpec, *, factories: dict | None = None) -> JobR
|
||||
with session_factory() as session:
|
||||
job_repo(session).create(job)
|
||||
session.commit()
|
||||
execute_job(
|
||||
job.id,
|
||||
session_factory=session_factory,
|
||||
job_repo_factory=job_repo,
|
||||
experiment_repo_factory=facts["experiment_repo_factory"],
|
||||
stock_repo_factory=facts["stock_repo_factory"],
|
||||
daily_repo_factory=facts["daily_repo_factory"],
|
||||
engine=facts["engine"],
|
||||
)
|
||||
if _job_mode() == "subprocess":
|
||||
_run_in_subprocess(job.id, session_factory=session_factory, job_repo_factory=job_repo)
|
||||
else:
|
||||
execute_job(
|
||||
job.id,
|
||||
session_factory=session_factory,
|
||||
job_repo_factory=job_repo,
|
||||
experiment_repo_factory=facts["experiment_repo_factory"],
|
||||
stock_repo_factory=facts["stock_repo_factory"],
|
||||
daily_repo_factory=facts["daily_repo_factory"],
|
||||
engine=facts["engine"],
|
||||
)
|
||||
with session_factory() as session:
|
||||
done = job_repo(session).get(job.id)
|
||||
assert done is not None
|
||||
return done
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 执行模式调度（config job.mode：local | subprocess）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _job_mode() -> str:
|
||||
from app.core.config import get_settings
|
||||
|
||||
return get_settings().job_mode
|
||||
|
||||
|
||||
def _job_memory_limit_gb() -> int:
|
||||
from app.core.config import get_settings
|
||||
|
||||
return get_settings().job_memory_limit_gb
|
||||
|
||||
|
||||
def _job_max_concurrent() -> int:
|
||||
from app.core.config import get_settings
|
||||
|
||||
return get_settings().job_max_concurrent
|
||||
|
||||
|
||||
def _job_worker_cmd(job_id: str) -> list[str]:
|
||||
"""研究子进程命令行（独立进程执行，cwd=backend 使 `-m app.cli.run_job` 可导入）。"""
|
||||
return [sys.executable, "-m", "app.cli.run_job", job_id]
|
||||
|
||||
|
||||
def run_job_background(
|
||||
job_id: str,
|
||||
*,
|
||||
session_factory: Callable | None = None,
|
||||
job_repo_factory: Callable | None = None,
|
||||
) -> None:
|
||||
"""API 后台任务入口：按 job.mode 调度 Job 执行。"""
|
||||
if _job_mode() == "subprocess":
|
||||
_run_in_subprocess(
|
||||
job_id, session_factory=session_factory, job_repo_factory=job_repo_factory
|
||||
)
|
||||
else:
|
||||
execute_job(job_id, **default_factories())
|
||||
|
||||
|
||||
def _acquire_slot(job_id: str) -> bool:
|
||||
with _active_lock:
|
||||
if len(_active_jobs) >= max(_job_max_concurrent(), 1):
|
||||
return False
|
||||
_active_jobs[job_id] = None
|
||||
return True
|
||||
|
||||
|
||||
def _release_slot(job_id: str) -> None:
|
||||
with _active_lock:
|
||||
_active_jobs.pop(job_id, None)
|
||||
|
||||
|
||||
def _mark_failed(
|
||||
job_id: str,
|
||||
error: str,
|
||||
*,
|
||||
session_factory: Callable | None = None,
|
||||
job_repo_factory: Callable | None = None,
|
||||
) -> None:
|
||||
"""把非终态 Job 标记 failed（子进程异常退出 / 并发超限时兜底，防永久 running）。"""
|
||||
if session_factory is None or job_repo_factory is None:
|
||||
facts = default_factories()
|
||||
session_factory = session_factory or facts["session_factory"]
|
||||
job_repo_factory = job_repo_factory or facts["job_repo_factory"]
|
||||
try:
|
||||
with session_factory() as session:
|
||||
repo = job_repo_factory(session)
|
||||
job = repo.get(job_id)
|
||||
if job is not None and job.status in (JobStatus.QUEUED, JobStatus.RUNNING):
|
||||
job.status = JobStatus.FAILED
|
||||
job.error = error
|
||||
job.finished_at = datetime.now()
|
||||
repo.update(job)
|
||||
session.commit()
|
||||
except Exception: # noqa: BLE001 —— 兜底标记失败自身异常不向上抛
|
||||
pass
|
||||
|
||||
|
||||
def _run_in_subprocess(
|
||||
job_id: str,
|
||||
*,
|
||||
session_factory: Callable | None = None,
|
||||
job_repo_factory: Callable | None = None,
|
||||
) -> None:
|
||||
"""在独立 python 进程执行 Job：子进程 RLIMIT 上限（防 OOM 整机），
|
||||
父进程等待；子进程异常退出（如被杀）时把 Job 补记 failed。"""
|
||||
if not _acquire_slot(job_id):
|
||||
_mark_failed(
|
||||
job_id,
|
||||
"系统繁忙：并发研究任务已达上限，请稍后重试",
|
||||
session_factory=session_factory,
|
||||
job_repo_factory=job_repo_factory,
|
||||
)
|
||||
return
|
||||
env = dict(os.environ)
|
||||
env["QLIB_JOB_MEM_LIMIT_GB"] = str(_job_memory_limit_gb())
|
||||
rc = -1
|
||||
already_marked = False
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
_job_worker_cmd(job_id),
|
||||
cwd=BACKEND_ROOT,
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
with _active_lock:
|
||||
_active_jobs[job_id] = proc
|
||||
rc = proc.wait()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_mark_failed(
|
||||
job_id,
|
||||
f"研究子进程启动失败: {type(exc).__name__}: {exc}",
|
||||
session_factory=session_factory,
|
||||
job_repo_factory=job_repo_factory,
|
||||
)
|
||||
already_marked = True
|
||||
finally:
|
||||
_release_slot(job_id)
|
||||
if rc != 0 and not already_marked:
|
||||
_mark_failed(
|
||||
job_id,
|
||||
f"研究子进程异常退出(code={rc})：任务未完成",
|
||||
session_factory=session_factory,
|
||||
job_repo_factory=job_repo_factory,
|
||||
)
|
||||
|
||||
|
||||
def mark_stale_jobs_failed(
|
||||
*,
|
||||
session_factory: Callable | None = None,
|
||||
job_repo_factory: Callable | None = None,
|
||||
reason: str | None = None,
|
||||
) -> int:
|
||||
"""服务启动调用：把上次进程异常退出遗留的 queued/running Job 标记 failed。
|
||||
|
||||
返回处理数量。防「进程被杀后 Job 永久 running」（AGENT.md §20 状态机闭环）。
|
||||
"""
|
||||
facts = default_factories()
|
||||
sf = session_factory or facts["session_factory"]
|
||||
jr = job_repo_factory or facts["job_repo_factory"]
|
||||
counted = 0
|
||||
with sf() as session:
|
||||
repo = jr(session)
|
||||
for status in (JobStatus.QUEUED, JobStatus.RUNNING):
|
||||
for job in repo.list_by_status(status):
|
||||
job.status = JobStatus.FAILED
|
||||
job.error = reason or "服务重启：上次未完成任务被中断"
|
||||
job.finished_at = datetime.now()
|
||||
repo.update(job)
|
||||
counted += 1
|
||||
session.commit()
|
||||
return counted
|
||||
|
||||
Reference in New Issue
Block a user