perf(backend): 内存优化三项——全市场研究不再占满 8G

1) 数据装配流式+列裁剪:Repository 新增 stream_range_many_columns(只 SELECT
   所需列、SQL 侧转 REAL、yield_per 分批),引擎按 required_columns 取数
   (LocalEngine 仅 close+因子字段),消除 ORM/Decimal 全量物化;
2) 研究 Job 独立子进程执行(job.mode=subprocess):python -m app.cli.run_job
   在子进程内设 RLIMIT_AS 上限,OOM 归档 failed 而非拖垮 API worker;
   子进程异常退出由父进程补记 failed;并发上限 2;
3) 服务启动清理:残留 queued/running Job 标记 failed(防永久 running)。

实测同款全市场回测:uvicorn worker RSS 稳定 ~220MB,任务峰值内存由 4.1GB+
降至 ~470MB,24s 完成并归档(此前 43s 未完成即 OOM)。
新增/更新测试 96 passed,ruff 干净。
This commit is contained in:
Simon
2026-09-06 22:12:44 +08:00
parent 02e42184be
commit 195f5d41f4
18 changed files with 593 additions and 31 deletions
+49
View File
@@ -100,6 +100,55 @@ class TestDailyBarRepository:
assert repo.latest_date("600519.SH") == date(2024, 1, 4)
assert repo.latest_date("000001.SZ") is None
def test_stream_range_many_columns_subset_order_and_null(self, session: Session) -> None:
"""流式列裁剪:只返回所需数值列、SQL 侧转 float、按 symbol/trade_date 升序。"""
repo = SqlAlchemyDailyBarRepository(session)
bars = [
self._bar("2024-01-02"),
self._bar("2024-01-03"),
self._bar("2024-01-04"),
]
other = [
DailyBar(
symbol="000001.SZ",
trade_date=d.trade_date,
close=Decimal("9"),
volume=Decimal("1"),
)
for d in bars
]
repo.upsert_many([*bars, *other])
session.commit()
rows = list(
repo.stream_range_many_columns(
["600519.SH"], date(2024, 1, 2), date(2024, 1, 4), ["close", "volume"]
)
)
assert rows == [
("600519.SH", "2024-01-02", 100.5, 10000.0),
("600519.SH", "2024-01-03", 100.5, 10000.0),
("600519.SH", "2024-01-04", 100.5, 10000.0),
]
# NULL 数值 → None;白名单外列报错
null_bar = self._bar("2024-01-02").model_copy(update={"volume": None})
repo.upsert_many([null_bar])
session.commit()
rows2 = list(
repo.stream_range_many_columns(
["600519.SH"], date(2024, 1, 2), date(2024, 1, 2), ["volume"]
)
)
assert rows2 == [("600519.SH", "2024-01-02", None)]
with pytest.raises(ValueError):
list(
repo.stream_range_many_columns(
["600519.SH"], date(2024, 1, 2), date(2024, 1, 4), ["close", "nope"]
)
)
class TestFinancialRepository:
def _fin(self, announce: str, report: str = "2024-06-30") -> FinancialIndicator: