初始提交：高股息策略研究与回测系统

从 Point-in-Time 股票筛选到统一 Web 前端的完整链路：筛选 → 画像 → 策略 → 回测 → Walk-forward → 绩效分析 → 报告/前端。架构 - 数据层与策略层分离；策略代码不写 SQL，只经 data/repo.py 取数 - 所有业务阈值集中在 config/*.yml，代码零硬编码（字段写错直接报错） - 报告只做「run_id → SQL → 渲染」，不做任何计算，数字可追溯 - 前后端分离：output/ 静态站点 + hdiv web 提供的 REST API 数据安全 - 只增不删：SQL 钩子拦截 DELETE/DROP/TRUNCATE，并有源码扫描测试守护 - qlib 原有表只读，本项目数据写入 hd_ 前缀表 - 回补使用 INSERT IGNORE，保证既有行零改动 - .env 存密钥且已 gitignore；output/、logs/、.venv/ 不入库交付物 - 30 张 hd_* 表、7 个 YAML 配置、283 项自动化测试 - 统一 Web 前端（hash 路由 SPA）+ nginx 部署配置与 launchd 托管脚本如实声明的限制 - 策略缺少稳定的样本外超额收益（Walk-forward 7 窗口均值 -0.95%，基准 +2.29%）；其价值体现在回撤控制，而非超额收益 - 涨跌停/停牌约束仅覆盖 2019 年起；index_weight 尚未填充 - AI Agent 层（plan.md 第四版 P8）未实现详见 docs/user-guide.md 与 docs/implementation-status.md。
2026-10-03 13:54:56 +08:00
commit fce725e13c
127 changed files with 28001 additions and 0 deletions
@@ -0,0 +1,460 @@
+"""回测引擎、成本、分红、绩效与敏感性测试。
+
+覆盖的都是「错了也不会报错、只会静默给出错误结论」的地方：
+- 总市值漏掉现金 → 净值曲线失真；
+- 会计恒等式不平 → 成本/分红有遗漏；
+- 分位阈值口径混用 → 未来函数；
+- 分批建仓与减仓互相冲突 → 高频无效交易；
+- 参数耦合未同步 → 扫出的差异来自形状畸变而非阈值本身。
+"""
+
+from __future__ import annotations
+
+from datetime import date
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from hdiv.backtest.engine import (
+    CostModel,
+    Position,
+    Signal,
+    _months_between,
+    _round_lot,
+    reconcile,
+)
+from hdiv.backtest.walk_forward import WalkForwardRunner, _add_months, _add_years
+from hdiv.core.config import CostConfig, load_config
+from hdiv.strategy.registry import StrategyRegistry, apply_sweep, parse_sweep
+
+
+# ---------------------------------------------------------------------------
+# 成本模型
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def cost() -> CostModel:
+    return CostModel(load_config("cost"))
+
+
+def test_commission_has_minimum(cost: CostModel) -> None:
+    """小额成交必须触发最低佣金 5 元。"""
+    c, s, t = cost.fees(1000.0, "BUY")
+    assert c == pytest.approx(5.0), "1000 × 0.025% = 0.25 元，应被最低佣金托底"
+    assert s == 0.0, "买入不收印花税"
+    assert t == pytest.approx(1000.0 * 0.00001)
+
+
+def test_large_commission_uses_rate(cost: CostModel) -> None:
+    c, _, _ = cost.fees(1_000_000.0, "BUY")
+    assert c == pytest.approx(250.0)
+
+
+def test_stamp_duty_sell_only(cost: CostModel) -> None:
+    _, s_buy, _ = cost.fees(1_000_000.0, "BUY")
+    _, s_sell, _ = cost.fees(1_000_000.0, "SELL")
+    assert s_buy == 0.0
+    assert s_sell == pytest.approx(500.0)
+
+
+def test_slippage_direction(cost: CostModel) -> None:
+    """买入价上滑、卖出价下滑 —— 方向反了会凭空产生收益。"""
+    assert cost.slip(100.0, "BUY") > 100.0
+    assert cost.slip(100.0, "SELL") < 100.0
+    assert cost.slip(100.0, "BUY") == pytest.approx(100.1)  # 10bps
+
+
+def test_slippage_modes() -> None:
+    cfg = CostConfig.model_validate(
+        {"version": 1, "slippage": {"mode": "fixed", "value": 0.02}}
+    )
+    assert CostModel(cfg).slip(100.0, "BUY") == pytest.approx(100.02)
+    cfg2 = CostConfig.model_validate(
+        {"version": 1, "slippage": {"mode": "tick", "value": 2}}
+    )
+    assert CostModel(cfg2).slip(100.0, "BUY") == pytest.approx(100.02)
+
+
+def test_dividend_tax_by_holding_period(cost: CostModel) -> None:
+    """plan.md §30：持股越久税率越低，超过 1 年免税。"""
+    assert cost.dividend_tax_rate(10) == pytest.approx(0.20)
+    assert cost.dividend_tax_rate(100) == pytest.approx(0.10)
+    assert cost.dividend_tax_rate(400) == pytest.approx(0.00)
+
+
+# ---------------------------------------------------------------------------
+# A 股交易规则
+# ---------------------------------------------------------------------------
+
+
+def test_round_lot_is_100_shares() -> None:
+    assert _round_lot(150) == 100
+    assert _round_lot(99) == 0
+    assert _round_lot(1000) == 1000
+    assert _round_lot(-5) == 0
+
+
+def test_months_between() -> None:
+    assert _months_between(date(2024, 1, 15), date(2024, 1, 30)) == 0
+    assert _months_between(date(2024, 1, 15), date(2024, 2, 1)) == 1
+    assert _months_between(date(2023, 12, 1), date(2024, 12, 1)) == 12
+
+
+# ---------------------------------------------------------------------------
+# 资金对账（P4 硬验收）
+# ---------------------------------------------------------------------------
+
+
+class _T:
+    def __init__(self, side: str, amount: float, fee: float) -> None:
+        self.side = side
+        self.amount = amount
+        self.total_cost = fee
+
+
+def test_reconcile_balanced() -> None:
+    eq = pd.DataFrame({"cash": [1000.0, 300.0, 550.0]})
+    trades = [_T("BUY", 700.0, 5.0), _T("SELL", 300.0, 3.0)]
+    # 1000 - 700 - 5 - 3 + 300 + 分红 0 = 592?
+    # 实际：1000 - 700 - 5(买佣) - 3(卖佣) + 300 = 592；现金应为 592
+    eq = pd.DataFrame({"cash": [592.0]})
+    rc = reconcile(eq, trades, total_dividend_net=0.0, initial_capital=1000.0)
+    assert rc["balanced"] is True
+    assert rc["residual"] == pytest.approx(0.0)
+
+
+def test_reconcile_detects_missing_dividend() -> None:
+    eq = pd.DataFrame({"cash": [700.0]})
+    trades = [_T("BUY", 300.0, 0.0)]
+    rc = reconcile(eq, trades, total_dividend_net=0.0, initial_capital=1000.0)
+    # 期望现金 700，实际 700 → 平衡
+    assert rc["balanced"] is True
+    # 若真实有 50 元分红但没记账，现金会多出 50 → 应被检出
+    rc2 = reconcile(eq, trades, total_dividend_net=50.0, initial_capital=1000.0)
+    assert rc2["balanced"] is False
+    assert rc2["residual"] == pytest.approx(-50.0)
+
+
+def test_reconcile_does_not_include_position_value() -> None:
+    """买入的股票仍在账上，其市值不是现金口径的误差。"""
+    eq = pd.DataFrame({"cash": [200.0]})
+    trades = [_T("BUY", 800.0, 0.0)]
+    rc = reconcile(eq, trades, 0.0, 1000.0)
+    assert rc["balanced"] is True, "1000 − 800 = 200，持仓市值不应进入残差"
+
+
+# ---------------------------------------------------------------------------
+# 目标仓位阶梯（防「分批建仓/减仓互相冲突」）
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def engine():
+    from hdiv.backtest.engine import BacktestEngine
+
+    return BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
+
+
+def test_target_weight_ladder(engine) -> None:
+    """plan.md §19 的建仓阶梯 + §18 的减仓阶梯，必须合成单一函数。"""
+    assert engine._target_weight(95) == pytest.approx(1.00)
+    assert engine._target_weight(88) == pytest.approx(0.75)
+    assert engine._target_weight(82) == pytest.approx(0.50)
+    assert engine._target_weight(76) == pytest.approx(0.25)
+    assert engine._target_weight(45) == pytest.approx(0.50)
+    assert engine._target_weight(30) == pytest.approx(0.50)
+    assert engine._target_weight(10) == pytest.approx(0.00)
+
+
+def test_target_weight_has_dead_zone(engine) -> None:
+    """死区必须存在 —— 否则分位抖动会导致高频无效交易（原实现年换手 8.9）。"""
+    for pct in (51, 60, 70, 74.9):
+        assert engine._target_weight(pct) is None, f"分位 {pct} 应落在死区"
+
+
+def test_target_weight_is_monotonic_on_entry_side(engine) -> None:
+    """买入侧：分位越高仓位越重（单调不减）。"""
+    vals = [engine._target_weight(p) for p in (75, 80, 85, 90)]
+    assert all(v is not None for v in vals)
+    assert vals == sorted(vals)
+
+
+def test_target_weight_is_monotonic_on_exit_side(engine) -> None:
+    """卖出侧：分位越低仓位越轻（单调不减）。"""
+    vals = [engine._target_weight(p) for p in (10, 25, 40, 50)]
+    assert all(v is not None for v in vals)
+    assert vals == sorted(vals)
+
+
+def test_held_position_at_high_percentile_is_not_trimmed(engine) -> None:
+    """已持仓且分位很高时不应被误判为减仓 —— 这是原实现的真实 bug。"""
+    assert engine._target_weight(92) == pytest.approx(1.0)
+
+
+# ---------------------------------------------------------------------------
+# 参数扫描的耦合处理
+# ---------------------------------------------------------------------------
+
+
+def test_parse_sweep() -> None:
+    g = parse_sweep("entry.yield_percentile=70,75,80")
+    assert g == {"entry.yield_percentile": [70, 75, 80]}
+    g2 = parse_sweep("a.b=1,2;c.d=x,y")
+    assert g2 == {"a.b": [1, 2], "c.d": ["x", "y"]}
+
+
+def test_parse_sweep_rejects_bad_format() -> None:
+    from hdiv.core.errors import SchemaValidationError
+
+    with pytest.raises(SchemaValidationError):
+        parse_sweep("entry.yield_percentile")
+
+
+def test_apply_sweep_shifts_ladder_shape() -> None:
+    """扫描 entry.yield_percentile 时必须整体平移 scale_in，而非只改首档。"""
+    r = StrategyRegistry()
+    s = r.load("config/strategy/high_dividend_v1.yml")
+    original_weights = [st.weight for st in s.entry.scale_in]
+    for p in (65, 70, 80, 90, 95):
+        x = apply_sweep(s, {"entry.yield_percentile": p})
+        ladder = [st.percentile for st in x.entry.scale_in]
+        assert x.entry.yield_percentile == pytest.approx(ladder[0])
+        assert ladder == sorted(set(ladder)), f"P{p} 的阶梯必须严格升序：{ladder}"
+        assert max(ladder) <= 100, f"P{p} 的阶梯越界：{ladder}"
+        # 权重形状必须保持
+        assert [st.weight for st in x.entry.scale_in] == original_weights
+
+
+def test_apply_sweep_exit_coupling() -> None:
+    r = StrategyRegistry()
+    s = r.load("config/strategy/high_dividend_v1.yml")
+    x = apply_sweep(s, {"exit.yield_percentile": 30})
+    assert x.exit.yield_percentile == pytest.approx(30)
+    assert x.exit.scale_out[-1].percentile == pytest.approx(30)
+
+
+def test_apply_sweep_unknown_path() -> None:
+    from hdiv.core.errors import SchemaValidationError
+
+    r = StrategyRegistry()
+    s = r.load("config/strategy/high_dividend_v1.yml")
+    with pytest.raises(SchemaValidationError):
+        apply_sweep(s, {"entry.not_a_field": 1})
+
+
+# ---------------------------------------------------------------------------
+# 敏感性判读
+# ---------------------------------------------------------------------------
+
+
+def test_sensitivity_analysis_flags_spike() -> None:
+    """plan.md §27 的尖峰情形必须被识别为疑似过拟合。"""
+    from hdiv.analysis.sensitivity import SensitivityRunner
+
+    points = [
+        {"cagr": 0.13, "max_drawdown": -0.2},
+        {"cagr": 0.135, "max_drawdown": -0.2},
+        {"cagr": 0.20, "max_drawdown": -0.2},
+        {"cagr": 0.132, "max_drawdown": -0.2},
+        {"cagr": 0.128, "max_drawdown": -0.2},
+    ]
+    a = SensitivityRunner._analyse(points, {"entry.yield_percentile": [75]})
+    assert a["spikes"], "P80 的 20% 相对邻居是明显尖峰，必须被检出"
+    assert a["robust"] is False
+    assert "过拟合" in a["verdict"]
+
+
+def test_sensitivity_analysis_accepts_smooth_curve() -> None:
+    """plan.md §27 的平滑情形应被判定为对参数不敏感。"""
+    from hdiv.analysis.sensitivity import SensitivityRunner
+
+    points = [{"cagr": c, "max_drawdown": -0.2} for c in (0.13, 0.133, 0.135, 0.132, 0.130)]
+    a = SensitivityRunner._analyse(points, {"entry.yield_percentile": [75]})
+    assert not a["spikes"]
+    assert a["smoothness"] > 0.6
+    assert a["robust"] is True
+    assert "不敏感" in a["verdict"]
+
+
+# ---------------------------------------------------------------------------
+# Walk-forward 窗口
+# ---------------------------------------------------------------------------
+
+
+def test_walk_forward_windows_are_disjoint_and_ordered() -> None:
+    w = WalkForwardRunner()
+    wins = w.windows()
+    assert wins, "应至少切出一个窗口"
+    for a, b in zip(wins, wins[1:], strict=False):
+        assert a.test_end < b.test_start, "测试区间不得重叠"
+        assert a.test_start > a.train_end, "测试必须晚于训练"
+    for x in wins:
+        assert x.train_start < x.train_end < x.test_start <= x.test_end
+
+
+def test_walk_forward_freeze_is_enforced_by_config() -> None:
+    """plan.md §25：测试阶段禁止重新调参 —— 配置层必须拒绝关闭该开关。"""
+    cfg = load_config("backtest")
+    assert cfg.walk_forward.freeze_params_in_test is True
+
+
+def test_add_months_and_years() -> None:
+    assert _add_months(date(2024, 1, 31), 1) == date(2024, 2, 29), "闰年 2 月"
+    assert _add_months(date(2023, 1, 31), 1) == date(2023, 2, 28)
+    assert _add_months(date(2024, 12, 15), 1) == date(2025, 1, 15)
+    assert _add_years(date(2020, 2, 29), 1) == date(2021, 2, 28)
+
+
+# ---------------------------------------------------------------------------
+# 引擎端到端（小样本，含数据时执行）
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.db
+def test_engine_end_to_end_reconciliation() -> None:
+    """完整跑一段回测并验证资金对账必须平衡。"""
+    from hdiv.backtest.engine import BacktestEngine
+
+    db_ok = True
+    try:
+        from hdiv.data import db as _db
+
+        _db.load_dotenv_once()
+        _db.list_tables(load_config("datasource"))
+    except Exception:
+        db_ok = False
+    if not db_ok:
+        pytest.skip("数据库不可用")
+
+    engine = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
+    res = engine.run(start=date(2023, 1, 1), end=date(2024, 6, 28), persist=False, verbose=False)
+    rc = res["reconciliation"]
+    assert rc["balanced"], f"资金对账不平，残差 {rc['residual']}"
+    assert res["equity"]["nav"].iloc[0] == pytest.approx(1.0)
+
+    eq = res["equity"]
+    # 净值必须等于 总市值 / 期初资金
+    assert (eq["nav"] - eq["total_value"] / res["initial_capital"]).abs().max() < 1e-9
+    # 总市值必须等于 现金 + 持仓
+    assert (eq["total_value"] - (eq["cash"] + eq["position_value"])).abs().max() < 1e-6
+    # 回撤不得为正
+    assert eq["drawdown"].max() <= 1e-9
+
+
+@pytest.mark.db
+def test_engine_uses_next_open_no_lookahead() -> None:
+    """成交日必须晚于信号日（plan.md §6 无未来函数）。"""
+    from hdiv.backtest.engine import BacktestEngine
+
+    try:
+        from hdiv.data import db as _db
+
+        _db.load_dotenv_once()
+        _db.list_tables(load_config("datasource"))
+    except Exception:
+        pytest.skip("数据库不可用")
+
+    engine = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
+    res = engine.run(start=date(2023, 1, 1), end=date(2024, 6, 28), persist=False, verbose=False)
+    for t in res["trades"]:
+        assert t.execution_date > t.signal_date, (
+            f"{t.symbol} 成交日 {t.execution_date} 未晚于信号日 {t.signal_date}"
+        )
+
+
+@pytest.mark.db
+def test_engine_dividends_are_creditable() -> None:
+    """持有期间应确实收到现金分红（高股息策略的核心收益来源）。"""
+    from hdiv.backtest.engine import BacktestEngine
+
+    try:
+        from hdiv.data import db as _db
+
+        _db.load_dotenv_once()
+        _db.list_tables(load_config("datasource"))
+    except Exception:
+        pytest.skip("数据库不可用")
+
+    engine = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
+    res = engine.run(start=date(2021, 1, 1), end=date(2024, 6, 28), persist=False, verbose=False)
+    assert res["total_dividend_net"] > 0, "高股息策略在 3.5 年里不可能没有现金分红"
+    d = res["dividends"]
+    assert not d.empty
+    assert (d["net"] <= d["gross"] + 1e-9).all(), "税后不得大于税前"
+    assert (d["tax"] >= 0).all()
+
+
+@pytest.mark.db
+def test_engine_pit_discipline_reference_window(engine) -> None:
+    """rolling 参照窗口必须完全落在评估日之前（无未来函数）。"""
+    for day in (date(2022, 6, 30), date(2024, 1, 15)):
+        ref = engine._reference_window(day)
+        assert ref is not None
+        assert ref[1] == day, "参照窗口右端必须是评估日本身"
+        assert ref[0] < day
+        assert engine._reference_mode() == "rolling"
+
+
+def test_frozen_reference_overrides_rolling() -> None:
+    """frozen 模式下必须使用冻结窗口，不得回退到滚动窗口。"""
+    from hdiv.backtest.engine import BacktestEngine
+
+    s = StrategyRegistry().load("config/strategy/high_dividend_v1.yml")
+    frozen = (date(2015, 1, 1), date(2019, 12, 31))
+    e = BacktestEngine(s, frozen_reference=frozen)
+    assert e._reference_window(date(2021, 6, 30)) == frozen
+    assert e._reference_mode() == "frozen"
+
+
+def test_reference_window_end_is_evaluation_day() -> None:
+    """rolling 窗口的右边界必须是评估日 —— 否则会用未来数据。"""
+    from hdiv.backtest.engine import BacktestEngine
+
+    s = StrategyRegistry().load("config/strategy/high_dividend_v1.yml")
+    e = BacktestEngine(s)
+    day = date(2023, 5, 10)
+    lo, hi = e._reference_window(day)
+    assert hi == day
+    assert (day - lo).days == int(365.25 * e.bt_cfg.percentile_reference.lookback_years)
+
+
+# ---------------------------------------------------------------------------
+# 绩效指标
+# ---------------------------------------------------------------------------
+
+
+def test_compute_metrics_flags_insufficient_data() -> None:
+    from hdiv.analysis.performance import compute_metrics
+
+    m = compute_metrics(pd.DataFrame(), [], load_config("backtest"), "r1")
+    assert m == {}, "空曲线不应编造指标"
+
+    eq = pd.DataFrame({
+        "trade_date": [date(2024, 1, 2)],
+        "total_value": [1_000_000.0],
+        "daily_return": [0.0],
+        "drawdown": [0.0],
+        "cash": [1_000_000.0],
+        "position_value": [0.0],
+    })
+    m2 = compute_metrics(eq, [], load_config("backtest"), "r2")
+    assert m2["total_return"] == pytest.approx(0.0)
+    assert m2["max_drawdown"] == pytest.approx(0.0)
+
+
+def test_metrics_do_not_invent_values() -> None:
+    """样本不足时 Sharpe 必须为 None，而不是 0。"""
+    from hdiv.analysis.performance import compute_metrics
+
+    eq = pd.DataFrame({
+        "trade_date": [date(2024, 1, 2), date(2024, 1, 3)],
+        "total_value": [1_000_000.0, 1_010_000.0],
+        "daily_return": [0.0, 0.01],
+        "drawdown": [0.0, 0.0],
+        "cash": [0.0, 0.0],
+        "position_value": [1_000_000.0, 1_010_000.0],
+    })
+    m = compute_metrics(eq, [], load_config("backtest"), "r3")
+    assert m["sharpe"] is None, "1 个观测算不出波动率，Sharpe 必须是 None"