初始提交:高股息策略研究与回测系统

从 Point-in-Time 股票筛选到统一 Web 前端的完整链路:
筛选 → 画像 → 策略 → 回测 → Walk-forward → 绩效分析 → 报告/前端。

架构
- 数据层与策略层分离;策略代码不写 SQL,只经 data/repo.py 取数
- 所有业务阈值集中在 config/*.yml,代码零硬编码(字段写错直接报错)
- 报告只做「run_id → SQL → 渲染」,不做任何计算,数字可追溯
- 前后端分离:output/ 静态站点 + hdiv web 提供的 REST API

数据安全
- 只增不删:SQL 钩子拦截 DELETE/DROP/TRUNCATE,并有源码扫描测试守护
- qlib 原有表只读,本项目数据写入 hd_ 前缀表
- 回补使用 INSERT IGNORE,保证既有行零改动
- .env 存密钥且已 gitignore;output/、logs/、.venv/ 不入库

交付物
- 30 张 hd_* 表、7 个 YAML 配置、283 项自动化测试
- 统一 Web 前端(hash 路由 SPA)+ nginx 部署配置与 launchd 托管脚本

如实声明的限制
- 策略缺少稳定的样本外超额收益(Walk-forward 7 窗口均值 -0.95%,
  基准 +2.29%);其价值体现在回撤控制,而非超额收益
- 涨跌停/停牌约束仅覆盖 2019 年起;index_weight 尚未填充
- AI Agent 层(plan.md 第四版 P8)未实现

详见 docs/user-guide.md 与 docs/implementation-status.md。
This commit is contained in:
2026-10-03 13:54:56 +08:00
commit fce725e13c
127 changed files with 28001 additions and 0 deletions
+575
View File
@@ -0,0 +1,575 @@
"""Web 层测试:API 契约、软删除语义、站点归一化、路径重写。
**为什么要有契约测试**:前端(``web/app.js``)与后端(``web/server.py``)是
两个独立文件,改动其中一个很容易忘记另一个。这里把「前端会调用的接口」
写成断言 —— 只要后端删掉某个路由,测试立刻失败,而不是等到页面上报错。
"""
from __future__ import annotations
import json
import re
from pathlib import Path
import pytest
from hdiv.core.paths import project_root
from hdiv.web import site
from hdiv.web.server import ROUTES
# ---------------------------------------------------------------------------
# 前端 ↔ 后端 接口契约
# ---------------------------------------------------------------------------
#: 前端实际会调用的接口(method, 路径样例)
#: 路径样例用于匹配路由正则;改前端时需同步此表。
FRONTEND_CALLS: list[tuple[str, str]] = [
("GET", "/api/health"),
("GET", "/api/summary"),
("GET", "/api/universes"),
("GET", "/api/universes/abc123"),
("GET", "/api/universes/abc123/members"),
("GET", "/api/universes/abc123/backtests"),
("PATCH", "/api/universes/abc123"),
("GET", "/api/stocks/600519.SH"),
("GET", "/api/backtests"),
("GET", "/api/backtests/abc123"),
("GET", "/api/backtests/abc123/metrics"),
("GET", "/api/backtests/abc123/equity"),
("GET", "/api/backtests/abc123/trades"),
("GET", "/api/backtests/abc123/signals"),
("PATCH", "/api/backtests/abc123"),
]
def _match(method: str, path: str) -> bool:
return any(m == method and pat.match(path) for m, pat, _ in ROUTES)
@pytest.mark.parametrize("method,path", FRONTEND_CALLS)
def test_frontend_api_calls_have_routes(method: str, path: str) -> None:
assert _match(method, path), f"前端会调用 {method} {path},但后端没有对应路由"
def test_app_js_only_calls_known_api_prefixes() -> None:
"""app.js 里出现的接口路径必须在契约表中有覆盖。"""
js = (project_root() / "web" / "app.js").read_text(encoding="utf-8")
# 抓取 api(`xxx`) / api('xxx') / patch(`xxx`, ...) 的首段
calls = re.findall(r"\b(?:api|patch)\(\s*[`'\"]([a-z][\w/\-${}.\[\]]*)", js)
assert calls, "未从 app.js 中解析出任何接口调用(解析逻辑需更新)"
# FRONTEND_CALLS 里是绝对路径 /api/xxx;app.js 里的参数是相对 api 基址的
# (api('summary') 实际请求 /api/summary),因此要去掉 api/ 前缀再比较。
known = {
p.removeprefix("/api/").strip("/").split("/")[0]
for _m, p in FRONTEND_CALLS
}
unknown = {
c.split("/")[0].split("?")[0]
for c in calls
if c.split("/")[0].split("?")[0] not in known
}
# 允许 ${...} 模板片段(这些是动态拼出的 run_id / symbol,前缀已在表中)
unknown = {u for u in unknown if not u.startswith("$")}
assert not unknown, f"app.js 调用了契约表未覆盖的接口前缀:{sorted(unknown)}"
def test_members_endpoint_defaults_to_selected() -> None:
"""接口层默认值:不传 passed 时应等价于 passed=1,而非全部候选。"""
import inspect
from hdiv.web import server
src = inspect.getsource(server._members)
assert 'passed_raw == "all"' in src or "passed_raw == 'all'" in src, \
"默认值应只把显式的 all 当作「全部候选」"
assert 'passed_raw != "0"' in src or "passed_raw != '0'" in src, \
"未指定 passed 时应视为 1(仅入选)"
def test_every_route_has_a_frontend_or_cli_consumer() -> None:
"""反向检查:后端不应暴露无人使用的接口(便于发现遗留死接口)。"""
known_paths = {p for _m, p in FRONTEND_CALLS}
orphans = []
for _m, pat, fn in ROUTES:
# 用契约表中的样例路径试探该路由是否有消费者
sample = pat.pattern.replace("^", "").replace("$", "")
sample = re.sub(r"\(\?P<\w+>\[[^\]]+\]\+?\)", "abc123", sample)
if not any(pat.match(p) for p in known_paths) and "/stocks/" not in sample:
orphans.append((_m, pat.pattern))
# /stocks/ 由画像页使用;/universes/{id}/members/{sym} 为可选下钻
assert len(orphans) <= 2, f"疑似无人使用的接口:{orphans}"
# ---------------------------------------------------------------------------
# 资源路径重写(归档后图表不能失效)
# ---------------------------------------------------------------------------
def test_rewrite_asset_paths_depth0() -> None:
html = '<script src="assets/echarts.min.js"></script>'
assert site.rewrite_asset_paths(html, 0) == html
def test_rewrite_asset_paths_depth2() -> None:
html = '<script src="assets/echarts.min.js"></script>'
out = site.rewrite_asset_paths(html, 2)
assert 'src="../../assets/echarts.min.js"' in out
def test_rewrite_does_not_touch_absolute_or_external() -> None:
for html in (
'<script src="/assets/echarts.min.js"></script>',
'<script src="https://cdn.example.com/echarts.min.js"></script>',
'<script src="//cdn.example.com/echarts.min.js"></script>',
'<script src="../assets/echarts.min.js"></script>',
):
assert site.rewrite_asset_paths(html, 2) == html, f"不该改写:{html}"
def test_rewrite_handles_href_too() -> None:
html = '<link href="assets/app.css" rel="stylesheet">'
assert '../../assets/app.css' in site.rewrite_asset_paths(html, 2)
# ---------------------------------------------------------------------------
# 前端文件
# ---------------------------------------------------------------------------
def test_frontend_source_files_exist() -> None:
src = project_root() / "web"
for name in ("index.html", "app.css", "app.js"):
assert (src / name).is_file(), f"缺少前端文件 web/{name}"
def test_frontend_index_references_local_assets_only() -> None:
html = (project_root() / "web" / "index.html").read_text(encoding="utf-8")
refs = re.findall(r'(?:src|href)="([^"]+)"', html)
external = [r for r in refs if r.startswith(("http://", "https://", "//"))]
assert not external, f"前端引用了外部资源(违反离线约束):{external}"
assert "app/app.js" in refs and "app/app.css" in refs
assert "assets/echarts.min.js" in refs
def test_frontend_uses_hash_routing_only() -> None:
"""hash 路由是刻意的:nginx 只需托管静态文件,不必配置 rewrite。"""
js = (project_root() / "web" / "app.js").read_text(encoding="utf-8")
assert "location.hash" in js
# 不应出现 history.pushState 这类需要服务端配合的写法
assert "pushState" not in js
def test_frontend_escapes_html() -> None:
"""用户可输入记录名称/备注,必须转义以避免 XSS。"""
js = (project_root() / "web" / "app.js").read_text(encoding="utf-8")
assert "function esc(" in js or "const esc =" in js
assert "&quot;" in js and "&lt;" in js
# ---------------------------------------------------------------------------
# API 行为(需要数据库)
# ---------------------------------------------------------------------------
def _db_ready() -> bool:
try:
from hdiv.core.config import load_config
from hdiv.data import db
db.load_dotenv_once()
db.list_tables(load_config("datasource"))
return True
except Exception:
return False
requires_db = pytest.mark.skipif(not _db_ready(), reason="数据库不可用")
@requires_db
def test_service_summary_shape() -> None:
from hdiv.web import service
s = service.summary()
for k in ("universes", "backtests", "profiles", "db"):
assert k in s, f"summary 缺少字段 {k}"
@requires_db
def test_soft_delete_is_reversible_and_never_physical() -> None:
"""软删除语义:记录仍在库中,可通过 include_deleted 找回。"""
from hdiv.web import service
runs = service.list_universes()
if not runs:
pytest.skip("没有筛选记录可供测试")
rid = runs[0]["run_id"]
try:
service.update_universe(rid, {"deleted": True})
visible = {r["run_id"] for r in service.list_universes()}
assert rid not in visible, "软删除后不应出现在默认列表"
allruns = {r["run_id"] for r in service.list_universes(include_deleted=True)}
assert rid in allruns, "软删除的记录必须仍可通过 include_deleted 找回(不做物理删除)"
finally:
service.update_universe(rid, {"deleted": False})
assert rid in {r["run_id"] for r in service.list_universes()}
@requires_db
def test_archive_toggles_visibility() -> None:
from hdiv.web import service
runs = service.list_universes()
if not runs:
pytest.skip("没有筛选记录可供测试")
rid = runs[0]["run_id"]
try:
service.update_universe(rid, {"archived": True})
assert rid not in {r["run_id"] for r in service.list_universes()}
assert rid in {r["run_id"] for r in service.list_universes(include_archived=True)}
finally:
service.update_universe(rid, {"archived": False})
@requires_db
def test_rename_persists_and_nullable() -> None:
from hdiv.web import service
runs = service.list_universes()
if not runs:
pytest.skip("没有筛选记录可供测试")
rid = runs[0]["run_id"]
original = runs[0]["display_name"]
try:
got = service.update_universe(rid, {"display_name": "单元测试名称"})
assert got["display_name"] == "单元测试名称"
assert got["title"] == "单元测试名称"
# 清空后应回落到自动标题,而不是留空
got = service.update_universe(rid, {"display_name": ""})
assert got["display_name"] is None
assert got["title"], "清空命名后应回落到自动标题"
finally:
service.update_universe(rid, {"display_name": original or ""})
@requires_db
def test_universe_backtest_link_is_bidirectional() -> None:
from hdiv.web import service
runs = service.list_universes()
bts = service.list_backtests()
if not runs or not bts:
pytest.skip("缺少筛选记录或回测记录")
rid, bid = runs[0]["run_id"], bts[0]["run_id"]
original = bts[0]["universe_run_id"]
try:
service.update_backtest(bid, {"universe_run_id": rid})
linked = service.list_backtests(universe_run_id=rid)
assert bid in {x["run_id"] for x in linked}, "回测侧应能按股票池过滤到"
u = service.get_universe(rid)
assert u["linked_backtests"] >= 1, "股票池侧应统计到关联回测数"
finally:
service.update_backtest(bid, {"universe_run_id": original})
@requires_db
def test_link_rejects_nonexistent_universe() -> None:
from hdiv.web import service
bts = service.list_backtests()
if not bts:
pytest.skip("没有回测记录")
with pytest.raises(ValueError):
service.update_backtest(bts[0]["run_id"], {"universe_run_id": "不存在的runid"})
@requires_db
def test_update_rejects_empty_patch() -> None:
from hdiv.web import service
runs = service.list_universes()
if not runs:
pytest.skip("没有筛选记录")
with pytest.raises(ValueError):
service.update_universe(runs[0]["run_id"], {})
@requires_db
def test_all_api_payloads_are_json_serializable() -> None:
"""回归:pandas 的 numpy 标量曾导致接口 500。"""
from hdiv.web import service
from hdiv.web.server import _Encoder
runs = service.list_universes()
payloads: list[object] = [service.summary(), runs]
if runs:
rid = runs[0]["run_id"]
payloads += [
service.get_universe(rid),
service.list_members(rid, size=2),
service.list_backtests(universe_run_id=rid),
]
bts = service.list_backtests()
if bts:
bid = bts[0]["run_id"]
payloads += [
service.get_backtest(bid),
service.get_backtest_metrics(bid),
service.get_backtest_equity(bid),
service.list_backtest_trades(bid, size=2),
service.list_backtest_signals(bid),
]
for p in payloads:
if p is None:
continue
json.dumps(p, ensure_ascii=False, cls=_Encoder) # 不应抛异常
@requires_db
def test_reason_text_is_human_readable() -> None:
"""成交理由必须渲染成人话,而不是丢一坨 JSON 给前端。"""
from hdiv.web.service import _reason_text
txt = _reason_text({
"dividend_yield": 0.0575, "yield_percentile": 83.4,
"rule": "股息率历史分位 83.4% >= P75", "observation_count": 1200,
})
assert "股息率 5.75%" in txt
assert "历史分位 83.4%" in txt
assert "{" not in txt and "}" not in txt
# ---------------------------------------------------------------------------
# 部署资产(缺了它们就会出现「API 不可用」)
# ---------------------------------------------------------------------------
def test_nginx_example_has_api_proxy_before_static() -> None:
"""回归:nginx 示例必须包含 /api 反代,且排在静态规则之前。
漏掉反代块时,/ggx/api/health 会被当静态文件去 output/api/health 找 → 404,
页面能打开但显示「API 不可用」。
"""
cfg = (project_root() / "deploy" / "nginx.conf.example").read_text(encoding="utf-8")
proxy = cfg.find("proxy_pass")
assert proxy != -1, "nginx 示例缺少 proxy_pass(API 无法访问)"
# 反代块必须出现在静态 location 之前(nginx 前缀匹配取最长者,但顺位更易读且不易误删)
static = cfg.find("alias /srv/hddiv/site/")
assert static == -1 or proxy < static, "API 反代块应排在静态规则之前"
# location 与 proxy_pass 的末尾斜杠必须成对,否则路径会被改写错
assert re.search(r"location\s+/ggx/api/\s*\{", cfg)
assert "proxy_pass http://127.0.0.1:8099/api/;" in cfg
def test_launchd_template_is_valid_and_placeholder_based() -> None:
"""launchd 模板必须可渲染:占位符齐全、plist 结构完整。"""
import plistlib
tpl = (project_root() / "deploy" / "com.hddiv.web.plist.example")
assert tpl.is_file(), "缺少 launchd 模板"
raw = tpl.read_text(encoding="utf-8")
for ph in ("__PYTHON__", "__PROJECT_ROOT__"):
assert ph in raw, f"模板缺少占位符 {ph}"
# 注释里也有 __,解析时先剥掉注释
body = re.sub(r"<!--.*?-->", "", raw, flags=re.DOTALL)
data = plistlib.loads(body.encode("utf-8"))
assert data["Label"] == "com.hddiv.web"
assert data["RunAtLoad"] is True
assert data["KeepAlive"] == {"SuccessfulExit": False}, "应支持崩溃自愈"
assert "--api-only" in data["ProgramArguments"]
def test_install_service_script_placeholder_guard() -> None:
"""安装脚本必须检测未替换的占位符 —— 否则 launchd 会静默失败。"""
sh = (project_root() / "deploy" / "install-service.sh").read_text(encoding="utf-8")
assert "plutil -lint" in sh, "应校验 plist 格式"
assert "未替换的占位符" in sh or "grep -q \"__\"" in sh
assert "launchctl load" in sh and "launchctl unload" in sh
def test_serve_script_is_syntax_valid() -> None:
import subprocess
sh = project_root() / "deploy" / "serve.sh"
r = subprocess.run(["bash", "-n", str(sh)], capture_output=True, text=True)
assert r.returncode == 0, f"serve.sh 语法错误:{r.stderr}"
@requires_db
def test_members_default_returns_selected_only() -> None:
"""回归:不传 passed 时应返回「入选」股票,而不是全部候选。
曾默认返回全部候选(5000+ 只),与「股票清单」的语义不符。
"""
from hdiv.web import service
runs = service.list_universes()
if not runs:
pytest.skip("没有筛选记录")
rid = runs[0]["run_id"]
default = service.list_members(rid, passed=True, size=1)
allc = service.list_members(rid, passed=None, size=1)
rejected = service.list_members(rid, passed=False, size=1)
assert default["total"] <= allc["total"]
assert default["total"] + rejected["total"] == allc["total"], \
"入选数 + 淘汰数 应等于候选总数"
assert default["total"] > 0, "示例记录应有入选股票"
# ---------------------------------------------------------------------------
# HTML 报告降级为「导出件」
# ---------------------------------------------------------------------------
def test_html_flag_is_opt_in_in_cli() -> None:
"""回归:静态报告已降级为导出件 —— 默认不生成,需显式 --html。
改造后 SPA 是主界面,默认再产出 HTML 会:
1) 每次筛选多一个文件
2) 同 asof 重跑时按 asof 命名互相覆盖,与「每次运行都留痕」矛盾
"""
from hdiv.cli import build_parser
p = build_parser()
subs = {a.dest: a for a in p._actions if hasattr(a, "choices") and isinstance(a.choices, dict)}
sub = subs["command"].choices
for name in ("universe", "profile", "backtest", "audit", "sensitivity"):
args = p.parse_args([name] if name != "universe" else [name])
assert getattr(args, "html") is False, f"hdiv {name} 的 --html 应为 opt-in"
assert hasattr(args, "no_html"), f"hdiv {name} 应保留 --no-html 以免旧命令报错"
_ = sub # 断言子命令存在
def test_default_universe_run_writes_no_html(monkeypatch) -> None:
"""不传 --html 时不应调用 HTML 生成器。"""
from hdiv import cli
called = {"n": 0}
import hdiv.report.build as build
def fake(*a, **k):
called["n"] += 1
raise AssertionError("默认不应生成 HTML")
monkeypatch.setattr(build, "build_universe_report", fake)
# 仅验证解析结果:默认 html=False
p = cli.build_parser()
assert p.parse_args(["universe"]).html is False
assert p.parse_args(["universe", "--html"]).html is True
assert called["n"] == 0
def test_report_names_include_run_id() -> None:
"""报告文件名必须带执行 id,否则同 asof/同 symbol 重跑会互相覆盖。"""
from hdiv.core.config import load_config
n = load_config("report").naming
for key in ("universe", "profile", "backtest", "walkforward", "sensitivity"):
pattern = getattr(n, key)
assert "{run_id}" in pattern or "{wf_id}" in pattern or "{sens_id}" in pattern, \
f"naming.{key} 缺少执行 id 占位符,重跑会覆盖:{pattern}"
@requires_db
def test_universe_report_filenames_do_not_collide() -> None:
"""同 asof 的两条记录必须产出两个不同文件(实测回归)。"""
from hdiv.core.config import load_config
from hdiv.report.renderer import Renderer
r = Renderer()
a = r.name_from("universe", asof="2025-01-21", run_id="a" * 32)
b = r.name_from("universe", asof="2025-01-21", run_id="b" * 32)
assert a != b, "同 asof 不同 run 必须产生不同文件名"
assert load_config("report").naming.universe.startswith("reports/")
@requires_db
def test_universe_detail_exposes_chart_data() -> None:
"""前端图表所需数据由后端算好(前端不做业务计算)。"""
from hdiv.web import service
runs = service.list_universes()
if not runs:
pytest.skip("没有筛选记录")
u = service.get_universe(runs[0]["run_id"])
f = u["funnel"]
assert len(f["labels"]) == len(f["values"]) == 5
# 漏斗最后一段必须等于入选数,否则 stats 不自洽
assert f["values"][-1] == u["member_count"]
assert f["values"][0] == u["candidate_count"]
# 存活数必须单调不增
assert all(f["values"][i] >= f["values"][i + 1] for i in range(len(f["values"]) - 1)), \
f"漏斗存活数应单调不增:{f['values']}"
dist = u["industry_distribution"]
assert isinstance(dist, list)
if dist:
assert {"industry", "count"} <= set(dist[0])
assert dist[0]["count"] >= dist[-1]["count"], "行业分布应按数量降序"
# ---------------------------------------------------------------------------
# 站点根 index.html 的所有权(曾发生真实事故)
# ---------------------------------------------------------------------------
def test_naming_index_is_under_reports() -> None:
"""回归:静态报告索引曾与前端首页抢 output/index.html。
命名配置里 index 漏配 reports/ 前缀 + 调用点硬编码 "index.html",
导致跑一次 `hdiv audit --html`(内部会 build_index)就把前端首页
覆盖成静态报告索引。
"""
from hdiv.core.config import load_config
naming = load_config("report").naming
assert naming.index == "reports/index.html", \
f"naming.index 必须是 reports/index.html(当前 {naming.index})"
# 所有静态报告都应在 reports/ 下,不占用站点根
for key in ("index", "audit", "universe", "profile", "backtest",
"walkforward", "sensitivity"):
pattern = getattr(naming, key)
assert pattern.startswith("reports/"), \
f"naming.{key} 应输出到 reports/ 下,不占用站点根:{pattern}"
def test_renderer_refuses_to_write_site_root_index() -> None:
"""渲染器必须拒绝把报告写到站点根 index.html(前置拦截)。"""
from hdiv.core.errors import HdivError
from hdiv.report.renderer import Renderer
r = Renderer()
for bad in ("index.html", "./index.html"):
with pytest.raises(HdivError, match="统一前端首页冲突"):
r.render("reports/index.html", {}, bad, report_type="index")
def test_site_root_index_is_spa() -> None:
"""站点根 index.html 必须是前端外壳(引用 app/app.js),不是静态报告索引。"""
idx = project_root() / "output" / "index.html"
if not idx.is_file():
pytest.skip("尚未生成站点")
html = idx.read_text(encoding="utf-8")
assert "app/app.js" in html, "output/index.html 不是统一前端外壳(可能被报告覆盖)"
assert "报告索引" not in html or "app/app.js" in html
def test_build_index_uses_naming_config() -> None:
"""build_index 的输出路径必须来自 naming 配置,不能硬编码。"""
import inspect
from hdiv.report import build
src = inspect.getsource(build.build_index)
assert 'r.name_from("index")' in src, "build_index 应使用 naming 配置生成文件名"
# 不应再把 "index.html" 当输出名直接传进去
assert ' "index.html",\n' not in src, "build_index 仍在硬编码输出名"
def test_site_build_does_not_clobber_spa() -> None:
"""site.sync_frontend 之后,站点根 index.html 必须仍是 SPA。"""
from hdiv.web import site
site.sync_frontend(verbose=False)
html = (project_root() / "output" / "index.html").read_text(encoding="utf-8")
assert "app/app.js" in html