perf(backend): 内存优化三项——全市场研究不再占满 8G
1) 数据装配流式+列裁剪:Repository 新增 stream_range_many_columns(只 SELECT 所需列、SQL 侧转 REAL、yield_per 分批),引擎按 required_columns 取数 (LocalEngine 仅 close+因子字段),消除 ORM/Decimal 全量物化; 2) 研究 Job 独立子进程执行(job.mode=subprocess):python -m app.cli.run_job 在子进程内设 RLIMIT_AS 上限,OOM 归档 failed 而非拖垮 API worker; 子进程异常退出由父进程补记 failed;并发上限 2; 3) 服务启动清理:残留 queued/running Job 标记 failed(防永久 running)。 实测同款全市场回测:uvicorn worker RSS 稳定 ~220MB,任务峰值内存由 4.1GB+ 降至 ~470MB,24s 完成并归档(此前 43s 未完成即 OOM)。 新增/更新测试 96 passed,ruff 干净。
This commit is contained in:
@@ -100,6 +100,55 @@ class TestDailyBarRepository:
|
||||
assert repo.latest_date("600519.SH") == date(2024, 1, 4)
|
||||
assert repo.latest_date("000001.SZ") is None
|
||||
|
||||
def test_stream_range_many_columns_subset_order_and_null(self, session: Session) -> None:
|
||||
"""流式列裁剪:只返回所需数值列、SQL 侧转 float、按 symbol/trade_date 升序。"""
|
||||
repo = SqlAlchemyDailyBarRepository(session)
|
||||
bars = [
|
||||
self._bar("2024-01-02"),
|
||||
self._bar("2024-01-03"),
|
||||
self._bar("2024-01-04"),
|
||||
]
|
||||
other = [
|
||||
DailyBar(
|
||||
symbol="000001.SZ",
|
||||
trade_date=d.trade_date,
|
||||
close=Decimal("9"),
|
||||
volume=Decimal("1"),
|
||||
)
|
||||
for d in bars
|
||||
]
|
||||
repo.upsert_many([*bars, *other])
|
||||
session.commit()
|
||||
|
||||
rows = list(
|
||||
repo.stream_range_many_columns(
|
||||
["600519.SH"], date(2024, 1, 2), date(2024, 1, 4), ["close", "volume"]
|
||||
)
|
||||
)
|
||||
assert rows == [
|
||||
("600519.SH", "2024-01-02", 100.5, 10000.0),
|
||||
("600519.SH", "2024-01-03", 100.5, 10000.0),
|
||||
("600519.SH", "2024-01-04", 100.5, 10000.0),
|
||||
]
|
||||
|
||||
# NULL 数值 → None;白名单外列报错
|
||||
null_bar = self._bar("2024-01-02").model_copy(update={"volume": None})
|
||||
repo.upsert_many([null_bar])
|
||||
session.commit()
|
||||
rows2 = list(
|
||||
repo.stream_range_many_columns(
|
||||
["600519.SH"], date(2024, 1, 2), date(2024, 1, 2), ["volume"]
|
||||
)
|
||||
)
|
||||
assert rows2 == [("600519.SH", "2024-01-02", None)]
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
list(
|
||||
repo.stream_range_many_columns(
|
||||
["600519.SH"], date(2024, 1, 2), date(2024, 1, 4), ["close", "nope"]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class TestFinancialRepository:
|
||||
def _fin(self, announce: str, report: str = "2024-06-30") -> FinancialIndicator:
|
||||
|
||||
Reference in New Issue
Block a user