ggx/tests/test_backtest.py

"""回测引擎、成本、分红、绩效与敏感性测试。

覆盖的都是「错了也不会报错、只会静默给出错误结论」的地方：
- 总市值漏掉现金 → 净值曲线失真；
- 会计恒等式不平 → 成本/分红有遗漏；
- 分位阈值口径混用 → 未来函数；
- 分批建仓与减仓互相冲突 → 高频无效交易；
- 参数耦合未同步 → 扫出的差异来自形状畸变而非阈值本身。
"""

from __future__ import annotations

from datetime import date, timedelta

import numpy as np
import pandas as pd
import pytest

from hdiv.backtest.engine import (
    CostModel,
    Position,
    Signal,
    _months_between,
    _round_lot,
    build_yield_series,
    dedupe_dividend_events,
    dividend_handling_notes,
    reconcile,
)
from hdiv.backtest.walk_forward import WalkForwardRunner, _add_months, _add_years
from hdiv.core.config import CostConfig, load_config
from hdiv.strategy.registry import StrategyRegistry, apply_sweep, parse_sweep


# ---------------------------------------------------------------------------
# 成本模型
# ---------------------------------------------------------------------------


@pytest.fixture
def cost() -> CostModel:
    return CostModel(load_config("cost"))


def test_commission_has_minimum(cost: CostModel) -> None:
    """小额成交必须触发最低佣金 5 元。"""
    c, s, t = cost.fees(1000.0, "BUY")
    assert c == pytest.approx(5.0), "1000 × 0.025% = 0.25 元，应被最低佣金托底"
    assert s == 0.0, "买入不收印花税"
    assert t == pytest.approx(1000.0 * 0.00001)


def test_large_commission_uses_rate(cost: CostModel) -> None:
    c, _, _ = cost.fees(1_000_000.0, "BUY")
    assert c == pytest.approx(250.0)


def test_stamp_duty_sell_only(cost: CostModel) -> None:
    _, s_buy, _ = cost.fees(1_000_000.0, "BUY")
    _, s_sell, _ = cost.fees(1_000_000.0, "SELL")
    assert s_buy == 0.0
    assert s_sell == pytest.approx(500.0)


def test_slippage_direction(cost: CostModel) -> None:
    """买入价上滑、卖出价下滑 —— 方向反了会凭空产生收益。"""
    assert cost.slip(100.0, "BUY") > 100.0
    assert cost.slip(100.0, "SELL") < 100.0
    assert cost.slip(100.0, "BUY") == pytest.approx(100.1)  # 10bps


def test_slippage_modes() -> None:
    cfg = CostConfig.model_validate(
        {"version": 1, "slippage": {"mode": "fixed", "value": 0.02}}
    )
    assert CostModel(cfg).slip(100.0, "BUY") == pytest.approx(100.02)
    cfg2 = CostConfig.model_validate(
        {"version": 1, "slippage": {"mode": "tick", "value": 2}}
    )
    assert CostModel(cfg2).slip(100.0, "BUY") == pytest.approx(100.02)


def test_dividend_tax_by_holding_period(cost: CostModel) -> None:
    """plan.md §30：持股越久税率越低，超过 1 年免税。"""
    assert cost.dividend_tax_rate(10) == pytest.approx(0.20)
    assert cost.dividend_tax_rate(100) == pytest.approx(0.10)
    assert cost.dividend_tax_rate(400) == pytest.approx(0.00)


# ---------------------------------------------------------------------------
# A 股交易规则
# ---------------------------------------------------------------------------


def test_round_lot_is_100_shares() -> None:
    assert _round_lot(150) == 100
    assert _round_lot(99) == 0
    assert _round_lot(1000) == 1000
    assert _round_lot(-5) == 0


def test_months_between() -> None:
    assert _months_between(date(2024, 1, 15), date(2024, 1, 30)) == 0
    assert _months_between(date(2024, 1, 15), date(2024, 2, 1)) == 1
    assert _months_between(date(2023, 12, 1), date(2024, 12, 1)) == 12


# ---------------------------------------------------------------------------
# 资金对账（P4 硬验收）
# ---------------------------------------------------------------------------


class _T:
    def __init__(self, side: str, amount: float, fee: float) -> None:
        self.side = side
        self.amount = amount
        self.total_cost = fee


def test_reconcile_balanced() -> None:
    eq = pd.DataFrame({"cash": [1000.0, 300.0, 550.0]})
    trades = [_T("BUY", 700.0, 5.0), _T("SELL", 300.0, 3.0)]
    # 1000 - 700 - 5 - 3 + 300 + 分红 0 = 592?
    # 实际：1000 - 700 - 5(买佣) - 3(卖佣) + 300 = 592；现金应为 592
    eq = pd.DataFrame({"cash": [592.0]})
    rc = reconcile(eq, trades, total_dividend_net=0.0, initial_capital=1000.0)
    assert rc["balanced"] is True
    assert rc["residual"] == pytest.approx(0.0)


def test_reconcile_detects_missing_dividend() -> None:
    eq = pd.DataFrame({"cash": [700.0]})
    trades = [_T("BUY", 300.0, 0.0)]
    rc = reconcile(eq, trades, total_dividend_net=0.0, initial_capital=1000.0)
    # 期望现金 700，实际 700 → 平衡
    assert rc["balanced"] is True
    # 若真实有 50 元分红但没记账，现金会多出 50 → 应被检出
    rc2 = reconcile(eq, trades, total_dividend_net=50.0, initial_capital=1000.0)
    assert rc2["balanced"] is False
    assert rc2["residual"] == pytest.approx(-50.0)


def test_reconcile_does_not_include_position_value() -> None:
    """买入的股票仍在账上，其市值不是现金口径的误差。"""
    eq = pd.DataFrame({"cash": [200.0]})
    trades = [_T("BUY", 800.0, 0.0)]
    rc = reconcile(eq, trades, 0.0, 1000.0)
    assert rc["balanced"] is True, "1000 − 800 = 200，持仓市值不应进入残差"


# ---------------------------------------------------------------------------
# 行业排除清单：从 backtest.yml 一路走到选股滤网
#
# 本项目最怕的失效形态是「配置写了，但没有任何代码路径会读它」——
# 那样回测照跑、日志照打，只是排除从未生效。下面三个入口都必须接线。
# ---------------------------------------------------------------------------


def test_engine_applies_universe_exclusions_to_selector() -> None:
    from hdiv.backtest.engine import BacktestEngine
    from hdiv.core.config import load_config
    from hdiv.universe.selector import UniverseSelector

    expected = list(load_config("backtest").universe_exclusions.industries)
    eng = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
    assert eng.universe_cfg.industry_exclusions == expected
    market = UniverseSelector(eng.universe_cfg)._build_filters()["market"]
    assert market.exclude_industries == expected, "黑名单没有传到 market 滤网"


def test_walkforward_and_daily_runners_apply_universe_exclusions() -> None:
    """训练段校准 / 逐日选股各自都持有生效后的筛选配置。

    walk-forward 的训练段也要排除：否则冻结分布是在**含被排除行业**的池子上
    标定的，与测试段实际能买的池子口径不一致。
    """
    from hdiv.backtest.daily import DailyRunner
    from hdiv.backtest.walk_forward import WalkForwardRunner
    from hdiv.core.config import load_config

    expected = list(load_config("backtest").universe_exclusions.industries)
    path = "config/strategy/high_dividend_v1.yml"
    assert DailyRunner.from_strategy(path).universe_cfg.industry_exclusions == expected
    assert WalkForwardRunner.from_strategy(path).universe_cfg.industry_exclusions == expected


# ---------------------------------------------------------------------------
# 目标仓位阶梯（防「分批建仓/减仓互相冲突」）
# ---------------------------------------------------------------------------


@pytest.fixture
def engine():
    from hdiv.backtest.engine import BacktestEngine

    return BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")


def test_target_weight_ladder(engine) -> None:
    """plan.md §19 的建仓阶梯 + §18 的减仓阶梯，必须合成单一函数。"""
    assert engine._target_weight(95) == pytest.approx(1.00)
    assert engine._target_weight(88) == pytest.approx(0.75)
    assert engine._target_weight(82) == pytest.approx(0.50)
    assert engine._target_weight(76) == pytest.approx(0.25)
    assert engine._target_weight(45) == pytest.approx(0.50)
    assert engine._target_weight(30) == pytest.approx(0.50)
    assert engine._target_weight(10) == pytest.approx(0.00)


def test_target_weight_has_dead_zone(engine) -> None:
    """死区必须存在 —— 否则分位抖动会导致高频无效交易（原实现年换手 8.9）。"""
    for pct in (51, 60, 70, 74.9):
        assert engine._target_weight(pct) is None, f"分位 {pct} 应落在死区"


def test_target_weight_is_monotonic_on_entry_side(engine) -> None:
    """买入侧：分位越高仓位越重（单调不减）。"""
    vals = [engine._target_weight(p) for p in (75, 80, 85, 90)]
    assert all(v is not None for v in vals)
    assert vals == sorted(vals)


def test_target_weight_is_monotonic_on_exit_side(engine) -> None:
    """卖出侧：分位越低仓位越轻（单调不减）。"""
    vals = [engine._target_weight(p) for p in (10, 25, 40, 50)]
    assert all(v is not None for v in vals)
    assert vals == sorted(vals)


def test_held_position_at_high_percentile_is_not_trimmed(engine) -> None:
    """已持仓且分位很高时不应被误判为减仓 —— 这是原实现的真实 bug。"""
    assert engine._target_weight(92) == pytest.approx(1.0)


# ---------------------------------------------------------------------------
# 参数扫描的耦合处理
# ---------------------------------------------------------------------------


def test_parse_sweep() -> None:
    g = parse_sweep("entry.yield_percentile=70,75,80")
    assert g == {"entry.yield_percentile": [70, 75, 80]}
    g2 = parse_sweep("a.b=1,2;c.d=x,y")
    assert g2 == {"a.b": [1, 2], "c.d": ["x", "y"]}


def test_parse_sweep_rejects_bad_format() -> None:
    from hdiv.core.errors import SchemaValidationError

    with pytest.raises(SchemaValidationError):
        parse_sweep("entry.yield_percentile")


def test_apply_sweep_shifts_ladder_shape() -> None:
    """扫描 entry.yield_percentile 时必须整体平移 scale_in，而非只改首档。"""
    r = StrategyRegistry()
    s = r.load("config/strategy/high_dividend_v1.yml")
    original_weights = [st.weight for st in s.entry.scale_in]
    for p in (65, 70, 80, 90, 95):
        x = apply_sweep(s, {"entry.yield_percentile": p})
        ladder = [st.percentile for st in x.entry.scale_in]
        assert x.entry.yield_percentile == pytest.approx(ladder[0])
        assert ladder == sorted(set(ladder)), f"P{p} 的阶梯必须严格升序：{ladder}"
        assert max(ladder) <= 100, f"P{p} 的阶梯越界：{ladder}"
        # 权重形状必须保持
        assert [st.weight for st in x.entry.scale_in] == original_weights


def test_apply_sweep_exit_coupling() -> None:
    r = StrategyRegistry()
    s = r.load("config/strategy/high_dividend_v1.yml")
    x = apply_sweep(s, {"exit.yield_percentile": 30})
    assert x.exit.yield_percentile == pytest.approx(30)
    assert x.exit.scale_out[-1].percentile == pytest.approx(30)


def test_apply_sweep_unknown_path() -> None:
    from hdiv.core.errors import SchemaValidationError

    r = StrategyRegistry()
    s = r.load("config/strategy/high_dividend_v1.yml")
    with pytest.raises(SchemaValidationError):
        apply_sweep(s, {"entry.not_a_field": 1})


# ---------------------------------------------------------------------------
# 敏感性判读
# ---------------------------------------------------------------------------


def test_sensitivity_analysis_flags_spike() -> None:
    """plan.md §27 的尖峰情形必须被识别为疑似过拟合。"""
    from hdiv.analysis.sensitivity import SensitivityRunner

    points = [
        {"cagr": 0.13, "max_drawdown": -0.2},
        {"cagr": 0.135, "max_drawdown": -0.2},
        {"cagr": 0.20, "max_drawdown": -0.2},
        {"cagr": 0.132, "max_drawdown": -0.2},
        {"cagr": 0.128, "max_drawdown": -0.2},
    ]
    a = SensitivityRunner._analyse(points, {"entry.yield_percentile": [75]})
    assert a["spikes"], "P80 的 20% 相对邻居是明显尖峰，必须被检出"
    assert a["robust"] is False
    assert "过拟合" in a["verdict"]


def test_sensitivity_analysis_accepts_smooth_curve() -> None:
    """plan.md §27 的平滑情形应被判定为对参数不敏感。"""
    from hdiv.analysis.sensitivity import SensitivityRunner

    points = [{"cagr": c, "max_drawdown": -0.2} for c in (0.13, 0.133, 0.135, 0.132, 0.130)]
    a = SensitivityRunner._analyse(points, {"entry.yield_percentile": [75]})
    assert not a["spikes"]
    assert a["smoothness"] > 0.6
    assert a["robust"] is True
    assert "不敏感" in a["verdict"]


# ---------------------------------------------------------------------------
# Walk-forward 窗口
# ---------------------------------------------------------------------------


def test_walk_forward_windows_are_disjoint_and_ordered() -> None:
    w = WalkForwardRunner()
    wins = w.windows()
    assert wins, "应至少切出一个窗口"
    for a, b in zip(wins, wins[1:], strict=False):
        assert a.test_end < b.test_start, "测试区间不得重叠"
        assert a.test_start > a.train_end, "测试必须晚于训练"
    for x in wins:
        assert x.train_start < x.train_end < x.test_start <= x.test_end


def test_walk_forward_freeze_is_enforced_by_config() -> None:
    """plan.md §25：测试阶段禁止重新调参 —— 配置层必须拒绝关闭该开关。"""
    cfg = load_config("backtest")
    assert cfg.walk_forward.freeze_params_in_test is True


def test_add_months_and_years() -> None:
    assert _add_months(date(2024, 1, 31), 1) == date(2024, 2, 29), "闰年 2 月"
    assert _add_months(date(2023, 1, 31), 1) == date(2023, 2, 28)
    assert _add_months(date(2024, 12, 15), 1) == date(2025, 1, 15)
    assert _add_years(date(2020, 2, 29), 1) == date(2021, 2, 28)


# ---------------------------------------------------------------------------
# 引擎端到端（小样本，含数据时执行）
# ---------------------------------------------------------------------------


@pytest.mark.db
def test_engine_end_to_end_reconciliation() -> None:
    """完整跑一段回测并验证资金对账必须平衡。"""
    from hdiv.backtest.engine import BacktestEngine

    db_ok = True
    try:
        from hdiv.data import db as _db

        _db.load_dotenv_once()
        _db.list_tables(load_config("datasource"))
    except Exception:
        db_ok = False
    if not db_ok:
        pytest.skip("数据库不可用")

    engine = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
    res = engine.run(start=date(2023, 1, 1), end=date(2024, 6, 28), persist=False, verbose=False)
    rc = res["reconciliation"]
    assert rc["balanced"], f"资金对账不平，残差 {rc['residual']}"
    assert res["equity"]["nav"].iloc[0] == pytest.approx(1.0)

    eq = res["equity"]
    # 净值必须等于 总市值 / 期初资金
    assert (eq["nav"] - eq["total_value"] / res["initial_capital"]).abs().max() < 1e-9
    # 总市值必须等于 现金 + 持仓
    assert (eq["total_value"] - (eq["cash"] + eq["position_value"])).abs().max() < 1e-6
    # 回撤不得为正
    assert eq["drawdown"].max() <= 1e-9


@pytest.mark.db
def test_engine_uses_next_open_no_lookahead() -> None:
    """成交日必须晚于信号日（plan.md §6 无未来函数）。"""
    from hdiv.backtest.engine import BacktestEngine

    try:
        from hdiv.data import db as _db

        _db.load_dotenv_once()
        _db.list_tables(load_config("datasource"))
    except Exception:
        pytest.skip("数据库不可用")

    engine = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
    res = engine.run(start=date(2023, 1, 1), end=date(2024, 6, 28), persist=False, verbose=False)
    for t in res["trades"]:
        assert t.execution_date > t.signal_date, (
            f"{t.symbol} 成交日 {t.execution_date} 未晚于信号日 {t.signal_date}"
        )


# ---------------------------------------------------------------------------
# 分红与公司行为（plan.md §30/§31）
# ---------------------------------------------------------------------------


def _div_ctx(day: date, rows: list[dict]) -> dict:
    return {"div_by_date": {day: rows}}


def _held(
    symbol: str = "X.SH",
    *,
    quantity: float = 1000.0,
    price: float = 10.0,
    first_buy: date | None = None,
) -> Position:
    d = first_buy or date(2020, 1, 2)
    return Position(
        symbol=symbol, quantity=quantity, avg_cost=price, cost_basis=quantity * price,
        first_buy_date=d, last_buy_date=d,
    )


def test_pure_stock_dividend_is_not_dropped(engine) -> None:
    """10 送 10（无现金分红）必须照常调整股数，不得静默丢弃。

    纯送转的 ``cash_div_tax`` 是 NULL/0，而 ``stk_div`` > 0。价格是不复权价，
    除权日必然下跌；若按现金分红判空整行跳过，就会凭空记出一笔亏损。
    实测本库 2015-2026 区间内高股息池成员有 824 笔纯送转。
    """
    day = date(2024, 6, 20)
    pos = _held(quantity=1000.0, price=10.0)
    ledger: list[dict] = []
    cash = engine._apply_dividends(
        day,
        {"X.SH": pos},
        _div_ctx(day, [{"symbol": "X.SH", "cash_div_tax": None, "stk_div": 1.0,
                        "stk_bo_rate": 1.0, "stk_co_rate": None}]),
        0.0,
        ledger,
    )
    # 股价腰斩到 5 元、股数翻倍到 2000 股 → 市值不变
    assert pos.quantity == pytest.approx(2000.0)
    assert pos.quantity * 5.0 == pytest.approx(10000.0), "10 送 10 前后市值必须不变"
    assert pos.avg_cost == pytest.approx(5.0), "总成本不变，每股成本须随股数下降"
    assert cash == 0.0, "纯送转不产生现金"
    assert ledger[0]["shares_added"] == pytest.approx(1000.0)
    assert ledger[0]["stock_div_applied"] is True


def test_stock_dividend_written_as_zero_cash_is_applied(engine) -> None:
    """``cash_div_tax`` 写成 0（而非 NULL）的纯转增同样不能丢。"""
    day = date(2024, 6, 20)
    pos = _held()
    engine._apply_dividends(
        day,
        {"X.SH": pos},
        _div_ctx(day, [{"symbol": "X.SH", "cash_div_tax": 0.0, "stk_div": 0.5,
                        "stk_co_rate": 0.5}]),
        0.0,
        [],
    )
    assert pos.quantity == pytest.approx(1500.0)
    assert pos.avg_cost == pytest.approx(10000.0 / 1500.0)


def test_cash_and_stock_dividend_are_independent(engine) -> None:
    """同一行既有现金又有送转：两者都要入账，互不影响。"""
    day = date(2024, 6, 20)
    pos = _held(first_buy=day - timedelta(days=800))  # 持股 > 1 年 → 免征红利税
    ledger: list[dict] = []
    cash = engine._apply_dividends(
        day,
        {"X.SH": pos},
        _div_ctx(day, [{"symbol": "X.SH", "cash_div_tax": 0.5, "stk_div": 0.3,
                        "stk_bo_rate": 0.3}]),
        0.0,
        ledger,
    )
    assert cash == pytest.approx(500.0), "持股 > 1 年免征红利税，全额入账"
    assert pos.quantity == pytest.approx(1300.0)
    assert ledger[0]["gross"] == pytest.approx(500.0)
    assert ledger[0]["tax"] == pytest.approx(0.0)
    assert ledger[0]["shares_added"] == pytest.approx(300.0)


def test_empty_dividend_row_is_skipped(engine) -> None:
    """既无现金也无送转（数据异常行）才是该跳过的行，且不留账。"""
    day = date(2024, 6, 20)
    pos = _held()
    ledger: list[dict] = []
    engine._apply_dividends(
        day, {"X.SH": pos},
        _div_ctx(day, [{"symbol": "X.SH", "cash_div_tax": 0.0, "stk_div": 0.0}]),
        0.0, ledger,
    )
    assert ledger == []
    assert pos.quantity == pytest.approx(1000.0)


def test_dividend_cash_joins_the_investable_pool(engine) -> None:
    """分红现金必须与初始资金同一个现金池 —— 能直接用于买入，不被隔离。

    这是「分红再投资」的实际含义：除权日入账 → 下次调仓按目标权重再配置。
    """
    day = date(2024, 6, 20)
    positions = {"X.SH": _held("X.SH", quantity=10_000.0, price=1.0,
                               first_buy=day - timedelta(days=800))}  # 免税
    px = pd.DataFrame(
        {"open": [1.0], "close": [1.0]}, index=pd.DatetimeIndex([pd.Timestamp(day)])
    )
    ctx = {
        "div_by_date": {day: [{"symbol": "X.SH", "cash_div_tax": 0.10, "stk_div": None}]},
        "px_by_sym": {"X.SH": px, "Y.SH": px},
        "suspend": set(),
        "limits": {},
    }
    # 起点现金为 0：下面买得成，只可能来自这笔分红
    cash = engine._apply_dividends(day, positions, ctx, 0.0, [])
    assert cash == pytest.approx(1000.0), "10000 股 × 每股 0.10 元"

    sig = Signal(
        symbol="Y.SH", signal_date=day, kind="BUY", target_weight=0.10,
        yield_value=0.08, yield_percentile=80.0, price=None, reason={},
    )
    fill, cash_after, skip = engine._execute(sig, day, cash, positions, ctx, 0)
    assert fill is not None, f"分红现金未能用于买入：{skip}"
    assert fill.quantity > 0
    assert cash_after < cash
    assert cash_after >= 0.0


def test_duplicate_dividend_rows_are_credited_once(engine) -> None:
    """同一除权日的多条记录只入账一次（引擎在预载阶段按经济事件聚合）。

    实测 002352.SZ 2024-11-07 同时有 0.4 / 1.0 / 1.4 三条记录，逐行入账会把同一笔
    分红算两三次（现金与送转都会被放大）。
    """
    day = date(2024, 6, 20)
    raw = pd.DataFrame([
        {"symbol": "X.SH", "ex_date": day, "imp_ann_date": date(2024, 5, 20),
         "cash_div_tax": 0.5, "stk_div": None, "stk_bo_rate": None, "stk_co_rate": None},
        {"symbol": "X.SH", "ex_date": day, "imp_ann_date": date(2024, 6, 1),
         "cash_div_tax": 0.5, "stk_div": 0.4, "stk_bo_rate": None, "stk_co_rate": 0.4},
    ])
    ev = dedupe_dividend_events(raw)
    assert len(ev) == 1, "同一 (symbol, ex_date) 必须收敛成一笔"
    ctx = {"div_by_date": {day: ev.to_dict("records")}}
    pos = _held(first_buy=day - timedelta(days=800))  # 持股 > 1 年 → 免税
    cash = engine._apply_dividends(day, {"X.SH": pos}, ctx, 0.0, [])
    assert cash == pytest.approx(500.0), "同一笔现金分红只入账一次"
    assert pos.quantity == pytest.approx(1400.0), "送转只放大一次"


def test_dividend_handling_notes_match_each_mode() -> None:
    """声明口径必须与实现逐档对应：已实现的组合不得留声明，未实现的必须声明。

    背景：`cash_mode: reinvest` 的实际行为一直是「分红现金回落到可投资现金池、
    下次调仓按目标权重再配置」，却长期被声明成「未实现」—— 声明与行为两头都不准。
    """

    def notes(**kw) -> str:
        bt = load_config("backtest").model_copy(deep=True)
        for k, v in kw.items():
            setattr(bt.dividend, k, v)
        return " ".join(dividend_handling_notes(bt))

    # 已实现：可投资现金池 + 目标权重再配置，且不涉及未实现的配股
    assert notes(cash_mode="reinvest", reinvest_rule="portfolio_rebalance",
                 handle_stock_dividend=True, handle_rights_issue=False) == ""
    # 未实现：同股再投 / 永久留存 / 移出组合 / 不处理送转 / 配股，逐条都要声明
    assert "未实现 reinvest_rule" in notes(reinvest_rule="same_stock_next_open",
                                          handle_rights_issue=False)
    assert "cash_mode=hold" in notes(cash_mode="hold", handle_rights_issue=False)
    assert "cash_mode=cash_out" in notes(cash_mode="cash_out", handle_rights_issue=False)
    assert "handle_stock_dividend" in notes(handle_stock_dividend=False,
                                            handle_rights_issue=False)
    assert "配股" in notes(handle_rights_issue=True)


@pytest.mark.db
def test_engine_dividends_are_creditable() -> None:
    """持有期间应确实收到现金分红（高股息策略的核心收益来源）。"""
    from hdiv.backtest.engine import BacktestEngine

    try:
        from hdiv.data import db as _db

        _db.load_dotenv_once()
        _db.list_tables(load_config("datasource"))
    except Exception:
        pytest.skip("数据库不可用")

    engine = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
    res = engine.run(start=date(2021, 1, 1), end=date(2024, 6, 28), persist=False, verbose=False)
    assert res["total_dividend_net"] > 0, "高股息策略在 3.5 年里不可能没有现金分红"
    d = res["dividends"]
    assert not d.empty
    assert (d["net"] <= d["gross"] + 1e-9).all(), "税后不得大于税前"
    assert (d["tax"] >= 0).all()
    # 同一 (symbol, ex_date) 不得出现两笔入账 —— 库里同一除权日有多条 `实施`
    # 记录（全库 1401 组），引擎必须在预载阶段按经济事件聚合
    assert not d.duplicated(["ex_date", "symbol"]).any(), (
        f"同一除权日重复入账：{d[d.duplicated(['ex_date', 'symbol'], keep=False)]}"
    )


@pytest.mark.db
def test_engine_pit_discipline_reference_window(engine) -> None:
    """rolling 参照窗口必须完全落在评估日之前（无未来函数）。"""
    for day in (date(2022, 6, 30), date(2024, 1, 15)):
        ref = engine._reference_window(day)
        assert ref is not None
        assert ref[1] == day, "参照窗口右端必须是评估日本身"
        assert ref[0] < day
        assert engine._reference_mode() == "rolling"


def test_frozen_reference_overrides_rolling() -> None:
    """frozen 模式下必须使用冻结窗口，不得回退到滚动窗口。"""
    from hdiv.backtest.engine import BacktestEngine

    s = StrategyRegistry().load("config/strategy/high_dividend_v1.yml")
    frozen = (date(2015, 1, 1), date(2019, 12, 31))
    e = BacktestEngine(s, frozen_reference=frozen)
    assert e._reference_window(date(2021, 6, 30)) == frozen
    assert e._reference_mode() == "frozen"


def test_reference_window_end_is_evaluation_day() -> None:
    """rolling 窗口的右边界必须是评估日 —— 否则会用未来数据。"""
    from hdiv.backtest.engine import BacktestEngine

    s = StrategyRegistry().load("config/strategy/high_dividend_v1.yml")
    e = BacktestEngine(s)
    day = date(2023, 5, 10)
    lo, hi = e._reference_window(day)
    assert hi == day
    assert (day - lo).days == int(365.25 * e.bt_cfg.percentile_reference.lookback_years)


# ---------------------------------------------------------------------------
# 绩效指标
# ---------------------------------------------------------------------------


def test_compute_metrics_flags_insufficient_data() -> None:
    from hdiv.analysis.performance import compute_metrics

    m = compute_metrics(pd.DataFrame(), [], load_config("backtest"), "r1")
    assert m == {}, "空曲线不应编造指标"

    eq = pd.DataFrame({
        "trade_date": [date(2024, 1, 2)],
        "total_value": [1_000_000.0],
        "daily_return": [0.0],
        "drawdown": [0.0],
        "cash": [1_000_000.0],
        "position_value": [0.0],
    })
    m2 = compute_metrics(eq, [], load_config("backtest"), "r2")
    assert m2["total_return"] == pytest.approx(0.0)
    assert m2["max_drawdown"] == pytest.approx(0.0)


def test_metrics_do_not_invent_values() -> None:
    """样本不足时 Sharpe 必须为 None，而不是 0。"""
    from hdiv.analysis.performance import compute_metrics

    eq = pd.DataFrame({
        "trade_date": [date(2024, 1, 2), date(2024, 1, 3)],
        "total_value": [1_000_000.0, 1_010_000.0],
        "daily_return": [0.0, 0.01],
        "drawdown": [0.0, 0.0],
        "cash": [0.0, 0.0],
        "position_value": [1_000_000.0, 1_010_000.0],
    })
    m = compute_metrics(eq, [], load_config("backtest"), "r3")
    assert m["sharpe"] is None, "1 个观测算不出波动率，Sharpe 必须是 None"


# ---------------------------------------------------------------------------
# 分位参照的最小样本量保护
# ---------------------------------------------------------------------------


def test_min_observations_config_exists_with_sane_default() -> None:
    """回归：分位参考必须设最小样本量，否则退化分布会伪造 100% 分位。

    分位 = 「≤当前值的观测占比」。窗口里只有 1 个观测且恰好等于当前值时
    占比 100%，击穿任何买入阈值 —— 实测 2015-01-06（行情数据首日）
    8 只股票因此被「100% 分位」买入。
    """
    from hdiv.core.config import load_config

    ref = load_config("backtest").percentile_reference
    assert hasattr(ref, "min_observations"), "缺少 min_observations 配置"
    assert ref.min_observations >= 60, \
        f"最小样本量过低（{ref.min_observations}），至少应约一个季度"


def test_engine_guards_against_insufficient_reference_sample() -> None:
    """引擎必须在样本不足时跳过信号，而不是照常算分位。"""
    import inspect

    from hdiv.backtest import engine as eng

    src = inspect.getsource(eng)
    assert "min_observations" in src, "引擎未使用 min_observations"
    # 保护必须在计算 pct 之前，且以 continue 跳过该股当日
    i_guard = src.find("ref_ser.size < self.bt_cfg.percentile_reference.min_observations")
    i_pct = src.find("pct = float((ref_ser <= current)")
    assert i_guard != -1, "未找到最小样本量判断"
    assert i_pct != -1 and i_guard < i_pct, "样本量判断必须早于分位计算"
    assert "continue" in src[i_guard:i_pct], "样本不足应跳过（continue）而非降级计算"


@pytest.mark.db
def test_recent_backtests_have_no_weak_sample_trades() -> None:
    """按新配置跑出的回测不应存在弱样本成交（样本 < min_observations）。"""
    from hdiv.core.config import load_config
    from hdiv.data import db

    db.load_dotenv_once()
    cfg = load_config("datasource")
    min_obs = load_config("backtest").percentile_reference.min_observations
    df = db.read_sql(
        "SELECT run_id, COUNT(*) AS n FROM hd_backtest_trade "
        "WHERE JSON_EXTRACT(reason_json, '$.observation_count') IS NOT NULL "
        "  AND JSON_EXTRACT(reason_json, '$.observation_count') < :m "
        "  AND run_id IN (SELECT run_id FROM hd_backtest_run "
        "                 WHERE created_at > '2026-10-03 14:30:00') "
        "GROUP BY run_id",
        {"m": min_obs}, cfg=cfg,
    )
    assert df.empty, (
        f"存在弱样本成交的回测（应为 0）："
        f"{[(r['run_id'][:10], int(r['n'])) for _, r in df.iterrows()]}"
    )


# ---------------------------------------------------------------------------
# 实时画像闸门（entry.profile_gate）
# ---------------------------------------------------------------------------


def _trigger_ctx(sym: str = "000001.SZ", n: int = 280) -> dict:
    """构造一个「股息率处于历史最高分位」的最小上下文。

    每股分红恒定 1 元、股价从 20 元跌到 10 元 → 股息率从 5% 升到 10%，
    当前值即窗口最大值，分位 = 100% ≥ P75，必然触发买入条件。

    ``n`` 必须 **同时** 满足两个约束：
    - ≥ ``backtest.yml: percentile_reference.min_observations``（250，否则引擎跳过）；
    - ≈ ≤ 410 个自然日（TTM 分红窗口 365 + 宽限 45），否则序列尾部 TTM 分红归零、
      股息率变成 0、分位塌到 30% 以下，触发不了买入。280 个交易日 ≈ 392 天，两者都满足。
    """
    days = pd.bdate_range("2015-01-05", periods=n)
    close = np.concatenate([np.full(n - 50, 20.0), np.linspace(20.0, 10.0, 50)])
    px = pd.DataFrame({"open": close, "close": close}, index=days)
    ev = pd.DataFrame([{
        "ex_date": days[0], "imp_ann_date": days[0], "cash_div_tax": 1.0,
    }])
    px_by_sym = {sym: px}
    events = {sym: ev}
    # P2 之后引擎从 ctx["yield_by_sym"] 取预计算的股息率序列；这里用**同一个**
    # 生产函数构造，避免测试自己算一套（那就成了两套口径）。
    return {"px_by_sym": px_by_sym, "events": events,
            "yield_by_sym": build_yield_series(px_by_sym, events),
            "_last_day": days[-1].date()}


def _pass_gate(*_a, **_k) -> dict:
    return {"verdict": "PASS", "checks": [], "failed": [], "unverifiable": []}


def _reject_gate(*_a, **_k) -> dict:
    return {
        "verdict": "REJECT",
        "checks": [{"metric": "payout_ratio", "stat": "current_value", "op": "<=",
                    "threshold": 1.0, "actual": 1.4, "status": "OK",
                    "window_years": 0, "passed": False}],
        "failed": ["payout_ratio.current_value<=1"],
        "unverifiable": [],
    }


def test_gate_rejects_buy(engine, monkeypatch) -> None:
    """闸门不通过时必须改为 REJECT，且不产生 BUY。"""
    ctx = _trigger_ctx()
    day = ctx["_last_day"]
    monkeypatch.setattr(engine, "_gate", _reject_gate)
    sigs = engine._evaluate(day, 1e6, {}, {"000001.SZ"}, ctx)
    kinds = [s.kind for s in sigs]
    assert "BUY" not in kinds, f"闸门未拦住买入：{kinds}"
    assert kinds == ["REJECT"]
    rej = sigs[0]
    assert rej.reason["skip_reason"] == "PROFILE_GATE"
    assert rej.reason["executed"] is False
    # 「为什么不买」必须可追溯：逐条规则的实际值与阈值都要留下
    chk = rej.reason["profile_checks"]["payout_ratio"]
    assert chk["actual"] == pytest.approx(1.4) and chk["threshold"] == 1.0
    assert chk["passed"] is False


def test_gate_pass_keeps_buy(engine, monkeypatch) -> None:
    """闸门通过时买入必须照常发生（不能误杀）。"""
    ctx = _trigger_ctx()
    monkeypatch.setattr(engine, "_gate", _pass_gate)
    sigs = engine._evaluate(ctx["_last_day"], 1e6, {}, {"000001.SZ"}, ctx)
    kinds = [s.kind for s in sigs]
    assert kinds == ["BUY"], kinds


def test_gate_reject_leaves_existing_position_untouched(engine, monkeypatch) -> None:
    """闸门语义是「不值得买」，不是「该卖」—— 被拒时不得动已有仓位。"""
    ctx = _trigger_ctx()
    pos = {"000001.SZ": Position(symbol="000001.SZ", quantity=1000.0, avg_cost=15.0,
                                 first_buy_date=date(2015, 1, 5), last_buy_date=date(2015, 1, 5),
                                 cost_basis=15000.0)}
    monkeypatch.setattr(engine, "_gate", _reject_gate)
    sigs = engine._evaluate(ctx["_last_day"], 1e6, pos, {"000001.SZ"}, ctx)
    kinds = [s.kind for s in sigs]
    assert "TRIM" not in kinds and "SELL" not in kinds and "ADD" not in kinds, kinds
    assert kinds == ["REJECT"], "高仓位侧被拒时应只留痕，不调仓"


def test_gate_handles_unverifiable_conservatively(engine, monkeypatch) -> None:
    """无法验证（数据缺失/样本不足）时按配置保守处理，且理由要能区分。"""
    def _unver(*_a, **_k):
        return {"verdict": "REJECT", "checks": [
            {"metric": "roe_avg", "stat": "current_value", "op": ">=", "threshold": 0.08,
             "actual": None, "status": "INSUFFICIENT", "window_years": 0, "passed": None}],
            "failed": [], "unverifiable": ["roe_avg.current_value"]}

    ctx = _trigger_ctx()
    monkeypatch.setattr(engine, "_gate", _unver)
    sigs = engine._evaluate(ctx["_last_day"], 1e6, {}, {"000001.SZ"}, ctx)
    assert [s.kind for s in sigs] == ["REJECT"]
    assert "无法验证" in sigs[0].reason["rule"]
    assert "样本不足" in sigs[0].reason["reason_cn"]


def test_gate_disabled_returns_none(engine) -> None:
    """闸门关闭时 _gate 必须返回 None —— 调用方不产生任何额外行为。"""
    from hdiv.backtest.engine import BacktestEngine

    eng = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
    eng.gate_cfg.enabled = False
    eng.pit = object()  # 即使被注入也不得被使用
    assert eng._gate("000001.SZ", date(2018, 5, 18)) is None


def test_gate_enabled_but_uninitialized_fails_loudly() -> None:
    """启用但未初始化必须报错，不得静默放行（否则等于风控悄悄失效）。"""
    from hdiv.backtest.engine import BacktestEngine
    from hdiv.core.errors import HdivError

    eng = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
    eng.gate_cfg.enabled = True
    eng.pit = None
    with pytest.raises(HdivError) as ei:
        eng._gate("000001.SZ", date(2018, 5, 18))
    assert "_prepare" in str(ei.value)


@pytest.mark.db
def test_gate_enabled_backtest_records_rejections() -> None:
    """端到端：启用闸门的短区间回测必须留下可追溯的 REJECT 记录且资金对账平衡。"""
    from hdiv.backtest.engine import BacktestEngine

    try:
        eng = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
        assert eng.gate_cfg.enabled is True, "默认策略应已启用实时画像闸门"
        res = eng.run(start=date(2016, 1, 1), end=date(2016, 12, 31),
                      persist=False, verbose=False)
    except Exception as exc:  # 数据不可用
        pytest.skip(f"数据库不可用：{exc}")

    assert res["reconciliation"]["balanced"] is True
    stats = res["profile_gate"]
    assert stats["asof_contexts"] > 0, "应产生实时画像时点"
    rejects = [s for s in res["signals"] if s.kind == "REJECT"]
    assert rejects, "该区间应存在被画像剔除的买入信号"
    for s in rejects:
        assert s.reason["skip_reason"] == "PROFILE_GATE"
        gate = s.reason["profile_gate"]
        assert gate["verdict"] in {"REJECT", "UNVERIFIABLE"}
        assert gate["checks"], "每条 REJECT 都必须带逐规则留痕"
        assert gate["failed"] or gate["unverifiable"]
        for c in gate["checks"]:
            assert set(c) >= {"metric", "op", "threshold", "actual", "status", "passed"}


@pytest.mark.db
def test_unimplemented_declarations_are_honest() -> None:
    """自我声明必须两头都准：既不能漏报「配置写了但没实现」，
    也不能把「这批股票恰好没停牌」误报成「数据缺失」。

    背景：2026-10-04 之前，``unimplemented`` 用「过滤后集合为空」判定约束失效，
    于是 2026-08~09（hd_suspend 明明覆盖到 2026-09-30，只是这批股票没停牌）
    被声明成「hd_suspend 无数据」—— 把自己的建模正常状态说成数据缺陷。
    """
    from hdiv.backtest.engine import BacktestEngine

    try:
        eng = BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")
        res = eng.run(start=date(2026, 8, 3), end=date(2026, 9, 30),
                      persist=False, verbose=False)
    except Exception as exc:
        pytest.skip(f"数据库不可用：{exc}")

    decl = " ".join(res["unimplemented"])
    # ① 不得把「无停牌」误报成「无数据」（约束表在 2010 起有数据）
    assert "无数据" not in decl, f"误报数据缺失：{decl}"
    # ② 必须如实声明「配置承诺但未实现」的项
    for must in ("defer", "配股", "成交量占比"):
        assert must in decl, f"漏报未实现项 {must}：{decl}"
    # ③ 另一头也要准：**已实现**的组合不得留声明。当前配置
    #    （cash_mode=reinvest + reinvest_rule=portfolio_rebalance）的实际行为是
    #    「分红现金回落到可投资现金池、下次调仓按目标权重再配置」，声明它
    #    「未实现」会让使用者误以为分红现金被隔离成了不可投资资金。
    assert "分红再投资" not in decl, f"把已实现的分红再投资误报成未实现：{decl}"
    assert "reinvest_rule" not in decl, f"把已实现的再投资规则误报成未实现：{decl}"


# ---------------------------------------------------------------------------
# 卖出复核（修复 3b）：TTM 窗口台阶 vs 分红能力恶化
# ---------------------------------------------------------------------------


def _confirm_engine():
    """构造一个不取数的引擎实例：卖出复核只依赖 pending_div 与策略配置。"""
    from hdiv.backtest.engine import BacktestEngine

    return BacktestEngine.from_strategy("config/strategy/high_dividend_v1.yml")


def test_exit_confirmation_blocks_cashflow_timing_artefact() -> None:
    """实测 600690.SH 2026-07-30：可见现金只剩 0.26920，但 FY2025 年度 0.89151
    早在 2026-06-25 就已公告（除权 2026-08-21）→ 清仓必须被拦下。

    这就是用户报告的那笔「实际不该成交」的卖出：旧口径下 TTM 因重复记录虚高
    到 2.46848，随后塌到 0.53840/0.26920，把「现金流时点」误读成「分红能力恶化」。
    """
    eng = _confirm_engine()
    eng.pending_div = {"600690.SH": [(date(2026, 6, 25), 0.89151)]}
    ok, info = eng._exit_confirmed("600690.SH", date(2026, 7, 30), 0.26920)
    assert ok is False, "已公告未除权的分红足以抬高股息率，清仓应被判定为未确认"
    assert info["pending_dps"] == pytest.approx(0.89151)
    assert info["ratio"] < info["min_ratio"]
    assert eng.exit_unconfirmed == 1


def test_exit_confirmation_allows_real_dividend_cut() -> None:
    """真降息必须照常清仓：公告金额本身就低（或没有公告）时不得拦截。"""
    eng = _confirm_engine()
    # 无任何已公告未除权分红 → 股息率低就是低
    eng.pending_div = {}
    ok, info = eng._exit_confirmed("600690.SH", date(2026, 7, 30), 0.26920)
    assert ok is True
    assert info["pending_dps"] == 0.0

    # 公告的是一笔很小的分红（0.02），不足以把股息率抬高 → 仍应清仓
    eng2 = _confirm_engine()
    eng2.pending_div = {"X.SH": [(date(2026, 6, 25), 0.02)]}
    ok2, info2 = eng2._exit_confirmed("X.SH", date(2026, 7, 30), 0.26920)
    assert ok2 is True, f"小额公告不应拦住清仓，ratio={info2['ratio']}"


def test_exit_confirmation_is_pit_sensitive() -> None:
    """公告日之后才可见：公告日之前的那一天不得用未来公告去豁免清仓。"""
    eng = _confirm_engine()
    eng.pending_div = {"600690.SH": [(date(2026, 6, 25), 0.89151)]}
    # 公告前一天：当时确实不可知 → 照常清仓
    ok_before, info_before = eng._exit_confirmed(
        "600690.SH", date(2026, 6, 24), 0.26920
    )
    assert ok_before is True
    assert info_before["pending_dps"] == 0.0
    # 公告当天起可见
    ok_after, info_after = eng._exit_confirmed("600690.SH", date(2026, 6, 25), 0.26920)
    assert ok_after is False
    assert info_after["pending_dps"] == pytest.approx(0.89151)


def test_exit_confirmation_can_be_disabled() -> None:
    """关掉开关时行为与修复前逐字一致（回滚路径必须可用）。"""
    from hdiv.core.config import ExitConfig

    eng = _confirm_engine()
    eng.pending_div = {"600690.SH": [(date(2026, 6, 25), 0.89151)]}
    eng.strategy.exit = ExitConfig.model_validate(
        {"yield_percentile": 25, "scale_out": [], "confirm": {"enabled": False}}
    )
    ok, info = eng._exit_confirmed("600690.SH", date(2026, 7, 30), 0.26920)
    assert ok is True and info["enabled"] is False