feat: 股息率案例口径 + 策略库与图表统一 + 回测存档完整化

汇总三轮未提交的开发(每轮均在本机 MariaDB + 真实浏览器上验证):

1) 股息率案例(全市场股息率最高 n 只,默认 20,每 m 月择股)
   - 新增日频估值表 daily_basic + 迁移;股息率因子(dv_ratio / dividend_yield / TTM)
   - 名称历史表 stock_name_history:剔除 ST 按**择股日当时名称**判定,消除
     「曾高股息后 ST」的股息陷阱(实测 3.70pp 偏差)
   - 区间择股/调仓双周期(m 择股 / y 调仓)、指数成分与白名单、停牌近似剔除
   - 复权因子口径核对(4,164,742 行、缺失 0.0%)、收盘价成交与涨跌停拦单
   - 案例实测:2020-01-01~2026-09-04 总收益 +24.86%(年化 3.52%、回撤 -28.58%)

2) 策略库与前端统一
   - strategy 表 + CRUD/PUT 原地更新 + `describe_strategy` 按 spec 真实推导
     「一句话说明 + 计算公式 + 执行步骤 + 注意事项」(与引擎实执行规则同源)
   - 任何出现股票代码处都成对显示名称且可点击进个股页
   - 全站图表基座统一 TradingView Lightweight Charts(ECharts 依赖、
     锁文件、组件与文档标注一并清除),买卖点标记只落在真实交易日上

3) 回测存档完整化(可往复查看)
   - 同步端点(POST /api/backtests、/api/factor-tests)此前完全不落库 → 现在同样归档,
     归档 id 经响应头 X-Experiment-Id 返回(不破坏 response_model)
   - data_version 首次真实写入(数据快照指纹:最新交易日 + 各表规模)
   - 个股收益曲线默认**全量保存**(此前硬截断 60 只);超出体积预算才裁剪,
     并写 archive_meta(机器可读)+ unimplemented(人可读)如实标注
   - 列表 kind/q 过滤 + X-Total-Count(此前 limit=50 静默截断)、DELETE 归档
   - 只读归档页 /experiments/{id}(Server Component,SSR 直出**选股条件**与
     **交易执行依据**);结果视图按 kind 分发(backtest/factor_test/selection),
     非回测归档不套用回测口径
   - 新增 CLI:prune_experiments(保留策略,默认 dry-run)、
     restore_experiment_from_job(从 Job 副本按原 id 重建被删的历史归档,默认 dry-run)

门禁:pytest 388 passed、ruff All checks passed、tsc 0 错误、图表单测 7 passed、
next build 成功、契约脚本 verify_strategy_workspace 59/59(含按 kind 逐类验证归档页)。
This commit is contained in:
Simon
2026-09-20 07:31:04 +08:00
parent 7e15b7251e
commit 23972e7063
112 changed files with 17908 additions and 3893 deletions
@@ -241,6 +241,31 @@ class DailySymbolResult:
notes: list[str] = field(default_factory=list)
@dataclass
class DailyBasicDayResult:
"""单个交易日的每日指标同步结果(status: skip|ok|failed)。"""
trade_date: date
status: str
source: str | None = None # tushare(新浪不支持本接口)
rows_fetched: int = 0
rows_written: int = 0
notes: list[str] = field(default_factory=list)
@dataclass
class NameHistoryChunkResult:
"""单个时间分片的名称变更同步结果(status: ok|failed)。"""
start: date
end: date
status: str
source: str | None = None
rows_fetched: int = 0
rows_written: int = 0
notes: list[str] = field(default_factory=list)
# ---------------------------------------------------------------- 财务同步服务
class VerifiedFinancialSyncer:
@@ -577,6 +602,181 @@ class VerifiedDailySyncer:
)
# ---------------------------------------------------------------- 每日指标同步服务
class DailyBasicSyncer:
"""每日指标(估值 / 股息率 / 市值)同步:按交易日整表拉取。
设计(与财务/日线的「逐股校验兜底」不同):
- daily_basic 是**横截面整表**接口,Tushare 一次返回当日全市场,无法逐股兜底;
- 新浪不提供本接口 → 主源失败时**如实失败并写 sync_log**,禁止静默留缺口;
- 幂等键 (symbol, trade_date):重跑同日无副作用(upsert)。
- 交易日集合由日历仓储给出:只同步 is_open 且本地缺失的日期
(`missing_dates`),因此增量与断点续跑天然安全。
"""
def __init__(
self,
*,
primary: MarketDataProvider,
repo,
audit: Callable[[SyncLog], None],
batch_flush: int = 5,
) -> None:
self.primary = primary
self.repo = repo
self.audit = audit
self.batch_flush = max(batch_flush, 1)
def sync_day(self, trade_date: date) -> DailyBasicDayResult:
try:
rows = self.primary.get_daily_basic(trade_date)
except DataSourceAuthenticationError:
raise # 凭证/权限故障 → 快速失败(不要逐日重试把配额烧光)
except Exception as exc: # noqa: BLE001 —— 逐日失败不中断整段
_audit_sync(
self.audit,
source=self.primary.name,
api="get_daily_basic",
success=False,
reason=str(exc),
start=trade_date,
end=trade_date,
)
return DailyBasicDayResult(
trade_date=trade_date,
status="failed",
source=self.primary.name,
notes=[f"拉取失败: {exc}"],
)
# 过滤掉非法行(ts_code 缺失)—— 避免脏键污染幂等
rows = [r for r in rows if r.symbol and r.symbol != "None"]
written = self.repo.upsert_many(rows)
_audit_sync(
self.audit,
source=self.primary.name,
api="get_daily_basic",
success=True,
row_count=len(rows),
start=trade_date,
end=trade_date,
)
return DailyBasicDayResult(
trade_date=trade_date,
status="ok",
source=self.primary.name,
rows_fetched=len(rows),
rows_written=written,
)
def sync_range(
self,
start: date,
end: date,
*,
on_progress: Callable[[int, int, date], None] | None = None,
should_stop: Callable[[], bool] | None = None,
) -> list[DailyBasicDayResult]:
"""补齐 [start, end] 内开市但本地缺失的交易日。
on_progress(done, total, trade_date):进度回调(CLI 打印 / 日志)。
should_stop():返回 True 时提前收尾(优雅中断,已落库的行保持有效)。
"""
days = self.repo.missing_dates(start, end)
results: list[DailyBasicDayResult] = []
total = len(days)
for idx, day in enumerate(days, start=1):
if should_stop is not None and should_stop():
break
results.append(self.sync_day(day))
if on_progress is not None:
on_progress(idx, total, day)
return results
class NameHistorySyncer:
"""股票名称变更历史同步(Tushare namechange)—— 时点 ST 判定的数据基础。
设计要点:
- **按年分片**:namechange 支持区间批量查询(2020+ 仅 4031 行),但全历史
(1990 起)会触及单次 6000 行上限被**静默截断**,因此按自然年分片调用,
每片独立审计与计数,超限时 Provider 会告警。
- **幂等键 (symbol, start_date)**:重跑无副作用(upsert),可安全续跑。
- 新浪不提供本接口 → 主源失败如实写 sync_log(禁止静默留缺口,AGENT.md §24)。
- 为什么值得同步:`stock.name` 只是最新名称快照,用最新名称做 `exclude_st`
会把「曾为高股息、后变 ST/退市」的股息陷阱样本整段排除 ——
实测影响约 3.70pp 收益(见 docs/DEV_PLAN_DIVIDEND_BACKTEST.md §10.5)。
"""
def __init__(
self,
*,
primary: MarketDataProvider,
repo,
audit: Callable[[SyncLog], None],
) -> None:
self.primary = primary
self.repo = repo
self.audit = audit
def sync_chunk(self, start: date, end: date) -> NameHistoryChunkResult:
try:
rows = self.primary.get_name_changes(start, end)
except DataSourceAuthenticationError:
raise # 凭证/权限故障 → 快速失败,不要逐片重试烧配额
except Exception as exc: # noqa: BLE001 —— 单片失败不中断整段
_audit_sync(
self.audit,
source=self.primary.name,
api="get_namechange",
success=False,
reason=str(exc),
start=start,
end=end,
)
return NameHistoryChunkResult(
start=start, end=end, status="failed",
source=self.primary.name, notes=[f"拉取失败: {exc}"],
)
rows = [r for r in rows if r.symbol and r.symbol != "None"]
written = self.repo.upsert_many(rows)
_audit_sync(
self.audit,
source=self.primary.name,
api="get_namechange",
success=True,
row_count=len(rows),
start=start,
end=end,
)
return NameHistoryChunkResult(
start=start, end=end, status="ok", source=self.primary.name,
rows_fetched=len(rows), rows_written=written,
)
def sync_range(
self,
start: date,
end: date,
*,
on_progress: Callable[[int, int, date], None] | None = None,
) -> list[NameHistoryChunkResult]:
"""按自然年分片同步 [start, end](每片 1 次 API 调用)。"""
chunks: list[tuple[date, date]] = []
cursor = date(start.year, 1, 1)
while cursor <= end:
chunk_end = min(date(cursor.year, 12, 31), end)
chunks.append((max(cursor, start), chunk_end))
cursor = date(cursor.year + 1, 1, 1)
results: list[NameHistoryChunkResult] = []
total = len(chunks)
for idx, (cs, ce) in enumerate(chunks, start=1):
results.append(self.sync_chunk(cs, ce))
if on_progress is not None:
on_progress(idx, total, cs)
return results
# ---------------------------------------------------------------- 小工具
def _fin_key(row: FinancialIndicator) -> tuple:
@@ -0,0 +1,288 @@
"""Experiment 归档服务(AGENT.md §21「研究可复现」):同步与异步两条路径共用。
谁在用:
- 异步 Job:`job_executor._execute_inner` 执行成功后归档;
- 同步研究端点:`api/research.py` 的 `POST /backtests`、`POST /factor-tests`
(Phase 3 遗留的同步路径,此前只塞进程内存 `_LAST_*`,重启即丢,完全不落库)。
为什么单独成模块:`job_executor.py` 顶部依赖较重(研究服务、子进程调度),
把归档抽到此处既避免 `api/research.py` → `job_executor` 的循环导入风险,也让
「一次研究结果如何落库」只有一份实现(AGENT.md §40 简单可替换优先)。
归档内容:spec_json(复现依据)、完整结果 JSON、summary_text(人读摘要)、
code_version(git 短 rev)、**data_version(真实数据快照指纹,见 data_version 模块)**、
job_id(可追溯执行记录)、created_at。同一份结果**只在 experiment 存一份**:
- 异步 Job 路径下 `job.result_json` 不再重复写入(见 `job_executor`),
`GET /api/jobs/{id}` 经 `job.experiment_id` 回读 experiment;
- 同步端点路径下不存在 Job 记录,`job_id=None`。
大小护栏(诚实 + 不炸库):`experiment.result_json` 是 MEDIUMTEXT,**上限
16,777,215 字节**(MySQL 的 MEDIUMTEXT 按字节计,utf8mb4 下中文占 3 字节),
故预算按 UTF-8 字节执行,默认 12,000,000 字节(约上限的 71.5%,给行内其它
列、SQL 协议与未预料字段留余量)。超预算时按 |期末收益| 降序裁剪
`symbol_curves`,并把证据写进结果的 `archive_meta`(机器可读)与
`unimplemented`(人可读)——绝不静默丢弃。
"""
from __future__ import annotations
import json
import logging
import subprocess
import uuid
from datetime import datetime
from pathlib import Path
from app.domain.entities.research import ExperimentRecord
logger = logging.getLogger(__name__)
# backend/app/application/services/experiment_archive.py → parents[4] = 项目根
PROJECT_ROOT = Path(__file__).resolve().parents[4]
# 归档结果 JSON 的字节预算(可被 config research.archive_max_chars 覆盖)
DEFAULT_ARCHIVE_MAX_CHARS = 12_000_000
# MEDIUMTEXT 硬上限(字节),仅用于计算/说明默认预算的余量
MEDIUMTEXT_MAX_BYTES = 16_777_215
def new_id(prefix: str) -> str:
"""生成 `PREFIX-XXXXXXXX` 形式的业务主键(Job/Experiment/Selection 等共用)。"""
return f"{prefix}-{uuid.uuid4().hex[:8].upper()}"
def _git_short_rev() -> str | None:
"""当前代码版本(git 短 rev);无 git / 超时则如实返回 None。"""
try:
out = subprocess.run(
["git", "rev-parse", "--short", "HEAD"],
cwd=PROJECT_ROOT,
capture_output=True,
text=True,
timeout=5,
check=False,
)
return out.stdout.strip() or None
except Exception: # noqa: BLE001
return None
def _summary_text(kind: str, result) -> str | None:
"""人读摘要(列表页展示);未知 kind 返回 None 而不是编造。"""
from app.domain.entities.research import BacktestResult, FactorTestReport
from app.domain.entities.selection import SelectionResult
if kind == "backtest" and isinstance(result, BacktestResult):
s = result.summary
return (
f"总收益 {s.total_return_pct:.2f}% · 年化 {s.annual_return_pct:.2f}% · "
f"回撤 {s.max_drawdown_pct:.2f}%"
)
if isinstance(result, FactorTestReport):
return (
f"IC {result.ic_mean:.4f} · RankIC {result.rank_ic_mean:.4f} · "
f"样本 {result.sample_days} 日"
)
if kind == "selection" and isinstance(result, SelectionResult):
return (
f"as_of {result.as_of_date} · 选出 {result.statistics.selected} / "
f"评估 {result.statistics.evaluated}"
)
return None
def _archive_budget_bytes() -> int:
"""归档字节预算:config `research.archive_max_chars`(名义字符数)> 代码默认。"""
from app.core.config import get_settings
try:
configured = get_settings().research_archive_max_chars
except Exception as exc: # noqa: BLE001 —— 配置读不到不能阻断归档
logger.warning("归档预算配置读取失败,用默认值:%s: %s", type(exc).__name__, exc)
configured = None
value = int(configured) if configured else DEFAULT_ARCHIVE_MAX_CHARS
return value if value > 0 else DEFAULT_ARCHIVE_MAX_CHARS
def _curve_sort_key(curve: dict):
"""个股曲线排序键:|期末收益| 降序(与引擎默认输出顺序一致,保证可预期)。"""
try:
return abs(float(curve.get("final_return_pct") or 0.0))
except (TypeError, ValueError):
return 0.0
def _truncation_note(stored: int, total: int, budget_bytes: int) -> str:
"""裁剪说明(人可读):必须写清「实际存了多少 / 共多少 / 为什么」。"""
return (
f"归档体积超过预算({budget_bytes:,} 字节,MEDIUMTEXT 上限 16MB):"
f"个股收益曲线按 |期末收益| 降序仅存 {stored} / 共 {total} 只"
"(完整明细见 trades / signal_history;archive_meta.truncated=true)"
)
def _over_budget_note(budget_bytes: int, curves_total: int) -> str:
"""「曲线全裁掉仍超预算」的说明:主体(trades/positions/signal_history 等)
无法裁剪,如实标注而不是假装达标。"""
return (
f"归档主体(trades / positions / signal_history 等,期内持有 {curves_total} 只)"
f"已超过配置的体积预算({budget_bytes:,} 字节):个股收益曲线裁剪至 0 只仍无法达标,"
"本次仍尝试完整落库;若 MEDIUMTEXT 写入失败会如实报错(archive_meta.over_budget=true)"
)
def _fit_payload(payload: dict, *, budget_bytes: int) -> tuple[dict, str]:
"""给结果 payload 写入 `archive_meta`,必要时按预算裁剪 symbol_curves。
返回 (payload, 已序列化的 result_json)。裁剪判定全部作用于「最终字符串的
UTF-8 字节数」,不是估算:每次候选都真实序列化一次(超预算只发生在
异常巨大的归档上,此时多几次序列化的代价可接受)。
"""
has_curves = isinstance(payload.get("symbol_curves"), list)
curves: list[dict] = list(payload.get("symbol_curves") or [])
total = len(curves)
ordered = sorted(curves, key=_curve_sort_key, reverse=True)
notes = list(payload.get("unimplemented") or [])
def build(kept: list[dict], *, over_budget: bool = False) -> tuple[dict, str, dict]:
"""按 kept 组装候选 payload 并序列化(含 archive_meta 自收敛)。"""
truncated = has_curves and len(kept) < total
candidate = dict(payload)
extra_notes: list[str] = []
if has_curves:
candidate["symbol_curves"] = kept
if truncated:
extra_notes.append(_truncation_note(len(kept), total, budget_bytes))
if over_budget:
extra_notes.append(_over_budget_note(budget_bytes, total))
if extra_notes:
candidate["unimplemented"] = [*notes, *extra_notes]
meta = {
"budget_chars": budget_bytes,
"budget_bytes": budget_bytes,
"over_budget": over_budget,
"result_chars": 0,
"result_bytes": 0,
}
if has_curves:
meta = {
"curves_stored": len(kept),
"curves_total": total,
"truncated": truncated,
**meta,
}
text = ""
# result_chars/result_bytes 会改变自身长度,迭代至收敛(通常 2 轮内)
for _ in range(8):
meta["result_chars"] = len(text)
meta["result_bytes"] = len(text.encode("utf-8"))
candidate["archive_meta"] = meta
text = json.dumps(candidate, ensure_ascii=False)
if meta["result_chars"] == len(text) and meta["result_bytes"] == len(
text.encode("utf-8")
):
break
return candidate, text, meta
if not has_curves: # 因子测试 / 选股等无曲线结果:不裁剪,只记录体积
fitted, text, meta = build([])
if meta["result_bytes"] > budget_bytes:
# 无曲线可裁:如实标注超预算,仍尝试落库(写入失败会如实报错)
logger.warning(
"归档超过预算且无曲线可裁剪:%d 字节 > %d 字节", meta["result_bytes"], budget_bytes
)
fitted, text, meta = build([], over_budget=True)
return fitted, text
fitted, text, meta = build(ordered)
if meta["result_bytes"] <= budget_bytes:
return fitted, text
# 超预算:二分找「能放进预算的最大曲线数」(降序保留收益绝对值最大的那些)
lo, hi, best = 0, total, 0
while lo <= hi:
mid = (lo + hi) // 2
_, candidate_text, _ = build(ordered[:mid])
if len(candidate_text.encode("utf-8")) <= budget_bytes:
best = mid
lo = mid + 1
else:
hi = mid - 1
fitted, text, meta = build(ordered[:best])
if meta["result_bytes"] > budget_bytes:
# 连 0 条曲线都放不下:主体(trades/positions/…)超预算,如实标注
fitted, text, meta = build(ordered[:best], over_budget=True)
logger.warning(
"归档超过预算:symbol_curves 裁剪为 %d/%d(预算 %d 字节,实际 %d 字节)",
meta.get("curves_stored"),
total,
budget_bytes,
meta.get("result_bytes"),
)
return fitted, text
def archive_experiment(
*,
session,
kind: str,
spec_json: str,
result,
job_id: str | None = None,
experiment_repo=None,
) -> ExperimentRecord:
"""把一次研究结果落为 Experiment 归档,返回归档记录(已 save + commit)。
参数:
- `session`:SQLAlchemy Session(由调用方持有生命周期;本函数负责 commit);
- `kind`:`backtest` / `factor_test` / `selection`;
- `spec_json`:完整研究 spec(复现依据);
- `result`:pydantic 结果对象(`model_dump(mode="json")` 序列化);
- `job_id`:来源 Job(同步端点没有 Job,留 None);
- `experiment_repo`:ExperimentRepository(不传则用 SqlAlchemy 实现;测试可注入假仓储)。
`data_version` 由 `compute_data_version(session)` 计算(真实数据指纹,取不到则
如实降级为 `unavailable`,见该模块 docstring)。
"""
from app.infrastructure.persistence.sqlalchemy.data_version import compute_data_version
payload = result.model_dump(mode="json")
budget_bytes = _archive_budget_bytes()
payload, result_json = _fit_payload(payload, budget_bytes=budget_bytes)
meta = payload.get("archive_meta") or {}
# 把归档元数据同步写回结果对象:同步端点的响应体因此也如实标注裁剪/超预算
#(AGENT.md §24:不假装支持 / 不静默降级)。没有该字段的结果类型跳过。
if hasattr(result, "archive_meta"):
result.archive_meta = meta
archived_notes = payload.get("unimplemented")
if isinstance(archived_notes, list) and hasattr(result, "unimplemented"):
# 归档 JSON 里的 unimplemented 才是权威(含归档侧追加的裁剪/超预算说明),
# 让同步响应体与归档内容保持一字不差
result.unimplemented = list(archived_notes)
experiment = ExperimentRecord(
id=new_id("EXP"),
kind=kind,
spec_json=spec_json,
result_json=result_json,
summary_text=_summary_text(kind, result),
code_version=_git_short_rev(),
data_version=compute_data_version(session),
job_id=job_id,
created_at=datetime.now(),
)
repo = experiment_repo if experiment_repo is not None else _default_experiment_repo(session)
repo.save(experiment)
session.commit()
return experiment
def _default_experiment_repo(session):
from app.infrastructure.persistence.sqlalchemy.repositories.jobs_impl import (
SqlAlchemyExperimentRepository,
)
return SqlAlchemyExperimentRepository(session)
@@ -18,15 +18,12 @@ import os
import subprocess
import sys
import threading
import uuid
from collections.abc import Callable
from datetime import datetime
from pathlib import Path
from app.application.services.experiment_archive import archive_experiment, new_id
from app.domain.entities.research import (
BacktestResult,
ExperimentRecord,
FactorTestReport,
JobRecord,
JobStatus,
ResearchSpec,
@@ -42,38 +39,6 @@ _active_jobs: dict[str, subprocess.Popen | None] = {}
_active_lock = threading.Lock()
def new_id(prefix: str) -> str:
return f"{prefix}-{uuid.uuid4().hex[:8].upper()}"
def _git_short_rev() -> str | None:
try:
out = subprocess.run(
["git", "rev-parse", "--short", "HEAD"],
cwd=PROJECT_ROOT,
capture_output=True,
text=True,
timeout=5,
check=False,
)
return out.stdout.strip() or None
except Exception: # noqa: BLE001
return None
def _summary_text(kind: str, result) -> str | None:
from app.domain.entities.selection import SelectionResult
if kind == "backtest" and isinstance(result, BacktestResult):
s = result.summary
return f"总收益 {s.total_return_pct:.2f}% · 年化 {s.annual_return_pct:.2f}% · 回撤 {s.max_drawdown_pct:.2f}%"
if isinstance(result, FactorTestReport):
return f"IC {result.ic_mean:.4f} · RankIC {result.rank_ic_mean:.4f} · 样本 {result.sample_days} 日"
if kind == "selection" and isinstance(result, SelectionResult):
return f"as_of {result.as_of_date} · 选出 {result.statistics.selected} / 评估 {result.statistics.evaluated}"
return None
def _execute_inner(
job_id: str,
*,
@@ -83,7 +48,22 @@ def _execute_inner(
stock_repo_factory: Callable,
daily_repo_factory: Callable,
engine,
basic_repo_factory: Callable | None = None,
financial_repo_factory: Callable | None = None,
index_repo_factory: Callable | None = None,
name_repo_factory: Callable | None = None,
) -> None:
"""执行 Job。
可选工厂的语义(AGENT.md §24:不做静默降级):
- `basic_repo_factory`:daily_basic(dv_ratio 等每日指标)—— 因子/条件引用时必需
- 复权(qfq/hfq)折算在行情仓储 SQL 内完成,无需 adjust_factor 工厂
- `financial_repo_factory`:财务表 —— 条件引用 fundamental.* 时必需
- `index_repo_factory`:指数成分 —— universe.index_code 时必需
- `name_repo_factory`:名称变更历史 —— universe.exclude_st 时点口径;未注入则
回退最新名称(旧行为),结果如实标注残余偏差
未注入且 spec 需要时,由 Service 抛出明确错误(而非返回空结果)。
"""
with session_factory() as session:
job_repo = job_repo_factory(session)
experiment_repo = experiment_repo_factory(session)
@@ -96,18 +76,34 @@ def _execute_inner(
session.commit()
try:
is_selection = job.kind == "selection"
basic_repo = basic_repo_factory(session) if basic_repo_factory else None
index_repo = index_repo_factory(session) if index_repo_factory else None
name_repo = name_repo_factory(session) if name_repo_factory else None
if is_selection:
from app.application.services.selection_service import SelectionService
from app.domain.entities.selection import SelectionQuery
spec = SelectionQuery.model_validate_json(job.spec_json)
service = SelectionService(
stock_repo_factory(session), daily_repo_factory(session)
stock_repo_factory(session),
daily_repo_factory(session),
financial_repo_factory(session) if financial_repo_factory else None,
index_repo=index_repo,
basic_repo=basic_repo,
name_repo=name_repo,
)
else:
spec = ResearchSpec.model_validate_json(job.spec_json)
service = ResearchService(
stock_repo_factory(session), daily_repo_factory(session), engine
stock_repo_factory(session),
daily_repo_factory(session),
engine,
index_repo=index_repo,
basic_repo=basic_repo,
financial_repo=(
financial_repo_factory(session) if financial_repo_factory else None
),
name_repo=name_repo,
)
def _set_stage(name: str) -> None:
@@ -130,20 +126,20 @@ def _execute_inner(
else:
result = service.run_factor_test(spec, on_stage=_set_stage)
result_json = json.dumps(result.model_dump(mode="json"), ensure_ascii=False)
experiment = ExperimentRecord(
id=new_id("EXP"),
experiment = archive_experiment(
session=session,
kind=job.kind,
spec_json=job.spec_json,
result_json=result_json,
summary_text=_summary_text(job.kind, result),
code_version=_git_short_rev(),
result=result,
job_id=job.id,
created_at=datetime.now(),
experiment_repo=experiment_repo,
)
experiment_repo.save(experiment)
job.result_json = result_json
# 完整结果**只在 experiment 存一份**(此前 job.result_json 与
# experiment.result_json 各存一份相同内容,完整存档后 2× 浪费)。
# GET /api/jobs/{id} 经 job.experiment_id 回读 experiment;
# 老记录(result_json 有值、experiment_id 为空)仍走 job 回退解码。
job.result_json = None
job.experiment_id = experiment.id
job.status = JobStatus.SUCCESS
job.error = None
@@ -173,6 +169,10 @@ def execute_job(
stock_repo_factory: Callable,
daily_repo_factory: Callable,
engine,
basic_repo_factory: Callable | None = None,
financial_repo_factory: Callable | None = None,
index_repo_factory: Callable | None = None,
name_repo_factory: Callable | None = None,
) -> None:
"""入口包装:任何未预期异常都将 Job 标记 failed(防止卡在 queued/running)。"""
try:
@@ -184,6 +184,10 @@ def execute_job(
stock_repo_factory=stock_repo_factory,
daily_repo_factory=daily_repo_factory,
engine=engine,
basic_repo_factory=basic_repo_factory,
financial_repo_factory=financial_repo_factory,
index_repo_factory=index_repo_factory,
name_repo_factory=name_repo_factory,
)
except Exception as exc: # noqa: BLE001
try:
@@ -202,12 +206,18 @@ def execute_job(
def default_factories() -> dict:
"""后台执行所需的独立 Session / Repository / 引擎装配(跨请求生命周期)。"""
from app.infrastructure.persistence.sqlalchemy.repositories.index_impl import (
SqlAlchemyIndexConstituentRepository,
)
from app.infrastructure.persistence.sqlalchemy.repositories.jobs_impl import (
SqlAlchemyExperimentRepository,
SqlAlchemyJobRepository,
)
from app.infrastructure.persistence.sqlalchemy.repositories.market_impl import (
SqlAlchemyDailyBarRepository,
SqlAlchemyDailyBasicRepository,
SqlAlchemyFinancialRepository,
SqlAlchemyStockNameHistoryRepository,
SqlAlchemyStockRepository,
)
from app.infrastructure.persistence.sqlalchemy.session import SessionLocal
@@ -219,6 +229,13 @@ def default_factories() -> dict:
"experiment_repo_factory": lambda s: SqlAlchemyExperimentRepository(s),
"stock_repo_factory": lambda s: SqlAlchemyStockRepository(s),
"daily_repo_factory": lambda s: SqlAlchemyDailyBarRepository(s),
# 新增装配(daily_basic / adjust_factor / 财务 / 指数成分):
# 缺失时对应能力(dv_ratio 因子、qfq/hfq 复权、fundamental 条件、指数成份池)
# 会由 Service 明确报错,绝不静默降级
"basic_repo_factory": lambda s: SqlAlchemyDailyBasicRepository(s),
"financial_repo_factory": lambda s: SqlAlchemyFinancialRepository(s),
"index_repo_factory": lambda s: SqlAlchemyIndexConstituentRepository(s),
"name_repo_factory": lambda s: SqlAlchemyStockNameHistoryRepository(s),
"engine": LocalEngine(),
}
@@ -252,9 +269,21 @@ def submit_and_run(spec: ResearchSpec, *, factories: dict | None = None) -> JobR
stock_repo_factory=facts["stock_repo_factory"],
daily_repo_factory=facts["daily_repo_factory"],
engine=facts["engine"],
basic_repo_factory=facts.get("basic_repo_factory"),
financial_repo_factory=facts.get("financial_repo_factory"),
index_repo_factory=facts.get("index_repo_factory"),
name_repo_factory=facts.get("name_repo_factory"),
)
with session_factory() as session:
done = job_repo(session).get(job.id)
# **仅内存**读透(不落库):完整结果只存 experiment 一份,但 submit_and_run 的
# 既有调用方(scripts/run_dividend_case.py、agent 工具)习惯从 job.result_json
# 取结果,这里按 experiment_id 回读一次填进返回对象,避免调用方静默拿到空结果。
# 数据库中的 job.result_json 仍然保持 NULL(P1:不重复存第二份)。
if done is not None and done.experiment_id and done.result_json is None:
exp = facts["experiment_repo_factory"](session).get(done.experiment_id)
if exp is not None:
done.result_json = exp.result_json
assert done is not None
return done
@@ -341,6 +370,21 @@ def _mark_failed(
pass
def _subprocess_log_target():
"""研究子进程 stderr 的落盘目标。
原先父进程用 DEVNULL,会吞掉子进程全部诊断(含 run_job 打印的内存上限设置失败
警告),使「如实提示未实现项」落空。改为追加写入 <项目根>/logs/job-subprocess.log。
记日志失败(无权限/磁盘满)绝不能导致 Job 失败 —— 此时退回 DEVNULL。
"""
try:
log_dir = BACKEND_ROOT.parent / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
return (log_dir / "job-subprocess.log").open("a", encoding="utf-8")
except OSError:
return subprocess.DEVNULL
def _run_in_subprocess(
job_id: str,
*,
@@ -361,13 +405,14 @@ def _run_in_subprocess(
env["QLIB_JOB_MEM_LIMIT_GB"] = str(_job_memory_limit_gb())
rc = -1
already_marked = False
errlog = _subprocess_log_target()
try:
proc = subprocess.Popen(
_job_worker_cmd(job_id),
cwd=BACKEND_ROOT,
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
stderr=errlog,
)
with _active_lock:
_active_jobs[job_id] = proc
@@ -381,6 +426,8 @@ def _run_in_subprocess(
)
already_marked = True
finally:
if errlog is not subprocess.DEVNULL:
errlog.close()
_release_slot(job_id)
if rc != 0 and not already_marked:
_mark_failed(
@@ -18,7 +18,7 @@ from app.domain.repositories.market import DailyBarRepository, StockRepository
from app.quant.selection import factor_columns
from app.quant.service import load_daily_df
from app.quant.signal import generate_signals
from app.quant.universe import filter_stocks, resolve_members
from app.quant.universe import filter_stocks, names_as_of, resolve_members
MAX_SYMBOLS = 40
MAX_DAYS = 90
@@ -30,10 +30,13 @@ class ReplayService:
stock_repo: StockRepository,
daily_repo: DailyBarRepository,
index_repo=None,
name_repo=None,
) -> None:
self._stock_repo = stock_repo
self._daily_repo = daily_repo
self._index_repo = index_repo
# 名称变更历史仓储:exclude_st 的时点口径(与选股/回测一致,v2 §25)
self._name_repo = name_repo
def replay(
self,
@@ -48,9 +51,12 @@ class ReplayService:
raise ValueError("Bar Replay 需要 universe.symbols 白名单(≤40 只),避免全市场长任务")
if len(symbols) > MAX_SYMBOLS:
raise ValueError(f"Bar Replay 白名单最多 {MAX_SYMBOLS} 只,当前 {len(symbols)}")
all_stocks = self._stock_repo.list()
name_at, _applied = names_as_of(all_stocks, start, self._name_repo)
stocks = filter_stocks(
self._stock_repo.list(), query.universe, as_of=start,
all_stocks, query.universe, as_of=start,
members=resolve_members(self._index_repo, query.universe, start),
name_at=name_at,
)
if not stocks:
return ReplayResult(start=start, end=end, top_n=top_n)
@@ -28,12 +28,36 @@ from app.quant.selection import (
run_condition_selection,
run_score_selection,
)
from app.quant.service import load_daily_df
from app.quant.universe import filter_stocks, resolve_members
from app.quant.service import (
load_basic_df,
load_daily_df,
merge_basic_into_daily,
split_factor_columns,
)
from app.quant.universe import filter_stocks, names_as_of, resolve_members
_FUNDAMENTAL_PREFIX = "fundamental."
def fill_candidate_names(result: SelectionResult, stocks: list) -> SelectionResult:
"""把股票池的 `symbol → name` 回填进候选股(展示增强,未命中保持 None)。
为什么在业务层做:名称是展示数据而非选股语义,引擎(quant/selection.py)
只做纯数值计算,不应感知名称;而本用例的 `stocks` 已是 universe 过滤后的
股票实体列表(天然带 name),在这里一次性建立映射即可,不必在各出口各自查库。
查不到名称的候选保持 None —— 前端按「名称未知」渲染,不伪造也不报错。
"""
if not result.candidates or not stocks:
return result
name_map = {s.symbol: s.name for s in stocks if getattr(s, "name", None)}
if not name_map:
return result
for cand in result.candidates:
if cand.name is None:
cand.name = name_map.get(cand.symbol)
return result
class SelectionService:
"""选股用例入口:select(query) → SelectionResult(当前或历史 as_of)。"""
@@ -43,17 +67,26 @@ class SelectionService:
daily_repo: DailyBarRepository,
financial_repo: FinancialRepository | None = None,
index_repo=None,
basic_repo=None,
name_repo=None,
) -> None:
self._stock_repo = stock_repo
self._daily_repo = daily_repo
self._financial_repo = financial_repo
self._index_repo = index_repo
# 每日指标仓储(daily_basic):score 因子/条件引用 dv_ratio 等列时使用
self._basic_repo = basic_repo
# 名称变更历史仓储:exclude_st 的时点口径(与回测口径一致,v2 §25)
self._name_repo = name_repo
def select(self, query: SelectionQuery) -> SelectionResult:
as_of = query.as_of or date.today()
all_stocks = self._stock_repo.list()
name_at, _applied = names_as_of(all_stocks, as_of, self._name_repo)
stocks = filter_stocks(
self._stock_repo.list(), query.universe, as_of=as_of,
all_stocks, query.universe, as_of=as_of,
members=resolve_members(self._index_repo, query.universe, as_of),
name_at=name_at,
)
if not stocks:
return self._run(query, pd.DataFrame(), stocks, as_of, financial={})
@@ -62,14 +95,19 @@ class SelectionService:
columns = sorted(factor_columns(query))
else:
columns = sorted(condition_needed_columns(query))
bar_cols, basic_cols = split_factor_columns(columns)
data_start = as_of - timedelta(days=query.warmup_days)
daily = load_daily_df(
self._daily_repo,
symbols,
as_of - timedelta(days=query.warmup_days),
data_start,
as_of,
columns,
adjust=query.price_adjustment,
sorted(bar_cols),
adjust="none",
price_adjust=query.price_adjustment,
)
if basic_cols:
daily = self._attach_basic(daily, symbols, data_start, as_of, sorted(basic_cols))
financial: dict[str, FinancialIndicator] = {}
if query.method == "condition" and self._uses_fundamental(query):
financial = self._load_financial(symbols, as_of)
@@ -77,6 +115,28 @@ class SelectionService:
# ---- 内部 ----
def _attach_basic(
self,
daily: pd.DataFrame,
symbols: list[str],
start: date,
end: date,
columns: list[str],
) -> pd.DataFrame:
"""并入 daily_basic 列(与 ResearchService 同一装配逻辑,保证 v2 §25 一致性)。"""
if self._basic_repo is None:
raise ValueError(
f"选股条件/因子需要每日指标列 {columns}(daily_basic),"
"但未注入 DailyBasicRepository。请检查 API 的依赖装配。"
)
basic = load_basic_df(self._basic_repo, symbols, start, end, columns)
if basic.empty:
raise ValueError(
f"daily_basic 表在 {start}~{end} 无数据,无法计算需要 {columns} 的因子/条件。"
"请先运行:python -m app.cli.sync daily_basic --start 20200101"
)
return merge_basic_into_daily(daily, basic)
def _run(
self,
query: SelectionQuery,
@@ -86,8 +146,12 @@ class SelectionService:
financial: dict[str, FinancialIndicator],
) -> SelectionResult:
if query.method == "score":
return run_score_selection(daily, query, as_of)
return run_condition_selection(daily, stocks, query, as_of, financial)
result = run_score_selection(daily, query, as_of)
else:
result = run_condition_selection(daily, stocks, query, as_of, financial)
# 名称只在业务层回填:引擎(quant/selection.py)保持纯符号计算,
# 而 `stocks` 是本用例已经装配好的股票池,天然带 name,无需再查库/join。
return fill_candidate_names(result, stocks)
@staticmethod
def _uses_fundamental(query: SelectionQuery) -> bool:
@@ -15,7 +15,7 @@ from app.domain.repositories.market import DailyBarRepository, StockRepository
from app.quant.selection import factor_columns
from app.quant.service import load_daily_df
from app.quant.signal import generate_signals
from app.quant.universe import filter_stocks, resolve_members
from app.quant.universe import filter_stocks, names_as_of, resolve_members
class SignalService:
@@ -24,16 +24,22 @@ class SignalService:
stock_repo: StockRepository,
daily_repo: DailyBarRepository,
index_repo=None,
name_repo=None,
) -> None:
self._stock_repo = stock_repo
self._daily_repo = daily_repo
self._index_repo = index_repo
# 名称变更历史仓储:exclude_st 的时点口径(与选股/回测一致,v2 §25)
self._name_repo = name_repo
def signal(self, query: SelectionQuery, rules: SignalRules) -> SignalResult:
as_of = query.as_of or date.today()
all_stocks = self._stock_repo.list()
name_at, _applied = names_as_of(all_stocks, as_of, self._name_repo)
stocks = filter_stocks(
self._stock_repo.list(), query.universe, as_of=as_of,
all_stocks, query.universe, as_of=as_of,
members=resolve_members(self._index_repo, query.universe, as_of),
name_at=name_at,
)
if not stocks:
return generate_signals(pd.DataFrame(), query, rules, as_of)