ggx/tests/test_daily.py

"""每日动态股票池回测（``--mode daily``）测试。

三条必须被锁定的性质：

1. **取数口径不分叉** —— ``PitRepo`` 在某个 asof 上返回的行情/每日指标/财务/
   分红，必须与直连数据库的 ``Repo`` **逐值一致**。它只改了「怎么取」，
   没有改「怎么算」；这是每日选股能跑得快却仍然可信的前提。
   与 ``tests/test_profile_pit.py`` 锁定「实时画像 = 批量画像」是同一个思路。

2. **预剪枝不改变最终入选** —— 市场/风险滤网的候选集预剪枝只剔除「在区间内
   不可能通过市场滤网」的股票（交易所/板块/上市年限/市值上界）。开与关必须
   选出**完全相同**的成员，否则它是「近似」而不是「等价」。

3. **动态池语义** —— 持仓掉出当日股票池时：``pool_exit_action=hold`` 只停止
   加仓（不清仓、仍按分位卖出），``sell`` 则清仓。买卖决策都会留下画像证据，
   与它是否在池内无关。

需要数据库的用例标记为 ``db``（无库时跳过）。
"""

from __future__ import annotations

from datetime import date, timedelta

import numpy as np
import pandas as pd
import pytest

from hdiv.core.config import DailyConfig, load_config
from hdiv.universe.daily import DailyUniverseScreener
from hdiv.backtest.daily import _chunks
from hdiv.backtest.engine import build_yield_series

# ---------------------------------------------------------------------------
# 非 DB：配置与纯函数
# ---------------------------------------------------------------------------


class TestDailyConfig:
    def test_defaults_are_conservative(self) -> None:
        """默认值必须让 daily 段「什么都没开」不影响既有模式。"""
        c = DailyConfig()
        assert c.universe_refresh_days == 1
        assert c.signal_frequency_days == 1
        assert c.pool_exit_action == "hold"
        assert c.profile_on_trade is True
        assert c.persist_daily_universe is True
        assert c.chunk_years == 1

    def test_backtest_yml_has_daily_block(self) -> None:
        bt = load_config("backtest")
        assert bt.daily.universe_refresh_days >= 1
        assert bt.daily.pool_exit_action in {"hold", "sell"}

    @pytest.mark.parametrize(
        "field", ["universe_refresh_days", "signal_frequency_days", "chunk_years"]
    )
    def test_non_positive_rejected(self, field: str) -> None:
        from hdiv.core.errors import SchemaValidationError

        with pytest.raises((SchemaValidationError, ValueError)):
            DailyConfig(**{field: 0})

    def test_bad_pool_exit_action_rejected(self) -> None:
        from hdiv.core.errors import SchemaValidationError

        with pytest.raises((SchemaValidationError, ValueError)):
            DailyConfig(pool_exit_action="liquidate")


class TestChunks:
    def test_single_year(self) -> None:
        days = [date(2024, 1, 2), date(2024, 3, 1), date(2024, 12, 31)]
        out = _chunks(days, 1)
        assert out == [days]

    def test_year_boundary_splits(self) -> None:
        days = [date(2023, 12, 29), date(2024, 1, 2), date(2024, 12, 31),
                date(2025, 1, 2)]
        out = _chunks(days, 1)
        assert [len(c) for c in out] == [1, 2, 1]
        assert out[0][0] == date(2023, 12, 29)
        assert out[2][0] == date(2025, 1, 2)

    def test_multi_year_chunk(self) -> None:
        days = [date(2020, 6, 1), date(2021, 6, 1), date(2022, 6, 1),
                date(2023, 6, 1), date(2024, 6, 1)]
        out = _chunks(days, 2)
        assert [len(c) for c in out] == [2, 2, 1]

    def test_empty(self) -> None:
        assert _chunks([], 1) == []


class TestEngineDefaultsAreOff:
    """新增的引擎参数默认值必须让 single / walkforward 行为逐字不变。"""

    def test_new_params_default_off(self) -> None:
        from hdiv.backtest.engine import BacktestEngine
        from hdiv.strategy.registry import StrategyRegistry

        s = StrategyRegistry().load("config/strategy/high_dividend_v1.yml")
        eng = BacktestEngine(s)
        assert eng.universe_by_refresh is None
        assert eng.universe_refresh_days is None
        assert eng.signal_frequency_days is None
        assert eng.pool_exit_action == "hold"
        assert eng.profile_on_trade is False

    def test_monthly_signal_path_preserved(self) -> None:
        """按月判定信号的原始语义必须仍在源码里（daily 只是新增分支）。"""
        import inspect

        from hdiv.backtest import engine as mod

        src = inspect.getsource(mod.BacktestEngine._simulate)
        assert "freq_days" in src, "daily 的按交易日分支丢失"
        assert "_months_between(" in src, "原有按月判定分支被删除"
        assert "day_index % max(1, int(freq_days))" in src

    def test_pool_exit_branches_present(self) -> None:
        import inspect

        from hdiv.backtest import engine as mod

        src = inspect.getsource(mod.BacktestEngine._evaluate)
        assert "in_universe" in src
        assert "pool_exit_action" in src
        assert "OUT_OF_UNIVERSE" in src
        assert "_trade_signal" in src, "买卖信号的画像留痕出口丢失"


class TestMetricFormatting:
    """短区间下指标不可计算时的格式化（**修掉一个既有的崩溃**）。"""

    def test_pct_and_num_tolerate_none(self) -> None:
        from hdiv.backtest.engine import _num, _pct

        assert _pct(None) == "—"
        assert _num(None) == "—"
        assert _pct(0.1234) == "12.34%"
        assert _num(1.5) == "1.50"
        assert _pct(float("nan")) == "—"

    def test_no_raw_metric_format_in_hot_prints(self) -> None:
        """回测的进度/汇总打印不得直接对可能为 None 的指标做 :.2% 格式化。

        实测：15 个交易日的回测里 ``sharpe`` 为 None，直接 ``:.2f`` 会抛
        ``TypeError`` 并以完整 traceback 结束 —— 把「指标不可计算」这个正常状态
        说成了程序缺陷。engine 与 cli 的打印必须走 ``_pct`` / ``_num``。
        """
        import inspect

        from hdiv import cli
        from hdiv.backtest import engine as eng

        for fn in (eng.BacktestEngine.run, cli.cmd_backtest):
            src = inspect.getsource(fn)
            for bad in ("['sharpe']:.2f", "['cagr']:.2%",
                        "['max_drawdown']:.2%", "['total_return']:.2%"):
                assert bad not in src, f"{fn.__qualname__} 仍在直接格式化 {bad}"


@pytest.mark.db
def test_pit_profile_caches_are_bounded() -> None:
    """长区间每日回测**不能**把每个时点的面板都留在内存里（否则必然 OOM）。

    每个 ``_AsOfContext`` 持有该 asof 可见的分红超集（约 4 MB）。1600 个决策日
    不设上限 ≈ 6 GB。这里用跨越一年的多个 asof 证明缓存是有界的，
    同时 ``distinct_asof`` 仍如实汇报**累计**涉及的时点数。
    """
    from hdiv.profile.pit import _MAX_ASOF_CONTEXTS, PitProfileService
    from hdiv.universe.pit import PitRepo

    try:
        rows = PitRepo().stock_master()
        syms = rows["symbol"].astype(str).head(3).tolist()
    except Exception as exc:  # pragma: no cover
        pytest.skip(f"数据库不可用：{exc}")
    if not syms:
        pytest.skip("数据库无股票")
    svc = PitProfileService(window_years=5)
    svc.prepare(syms, date(2019, 1, 1), date(2024, 12, 31))
    svc.configure({"roe_avg"})
    days = [date(2024, m, 15) for m in range(1, 13)]
    for d in days:
        for s in syms:
            svc.snapshot(s, d)
    assert len(svc._ctx) <= _MAX_ASOF_CONTEXTS, "时点面板缓存无上限 —— 长回测会 OOM"
    assert len(svc._snapshots) <= len(syms) * 2, "画像快照缓存无上限"
    assert svc.stats()["distinct_asof"] == len(days), (
        "distinct_asof 必须是累计值，不能用当前缓存条数冒充"
    )


# ---------------------------------------------------------------------------
# 前端数据源：决策时点实时画像
# ---------------------------------------------------------------------------


@pytest.mark.db
def test_stock_detail_exposes_decision_time_profile() -> None:
    """成交个股的**决策时点实时画像**必须能从接口取到。

    这是前端个股页「决策时点实时画像」卡片的数据源
    （``trades[].reason.profile``）。这条链路断了，页面那张卡片会静默变空 ——
    而「当时凭什么买」正是 daily 模式最该留下的证据。
    """
    from hdiv.data import db
    from hdiv.web import analysis

    cfg = load_config("datasource")
    try:
        run = db.read_sql(
            "SELECT run_id FROM hd_backtest_run WHERE mode = 'daily' "
            "ORDER BY created_at DESC LIMIT 1", cfg=cfg,
        )
    except Exception as exc:  # pragma: no cover
        pytest.skip(f"数据库不可用：{exc}")
    if run.empty:
        pytest.skip("库里还没有 mode=daily 的回测记录")
    run_id = str(run["run_id"].iloc[0])

    tr = db.read_sql(
        "SELECT symbol FROM hd_backtest_trade WHERE run_id = :r "
        "AND JSON_EXTRACT(reason_json, '$.profile') IS NOT NULL LIMIT 1",
        {"r": run_id}, cfg=cfg,
    )
    if tr.empty:
        pytest.skip(f"{run_id} 的成交里没有画像留痕（可能关闭了 profile_on_trade）")
    symbol = str(tr["symbol"].iloc[0])

    d = analysis.stock_detail(run_id, symbol, series=["close"])
    with_profile = [t for t in d["trades"] if (t.get("reason") or {}).get("profile")]
    assert with_profile, "接口未返回决策时点画像（前端卡片会变空）"
    prof = with_profile[0]["reason"]["profile"]
    # 快照必须带「当时的值」与「当时的窗口」，否则无法复核
    assert isinstance(prof.get("values"), dict) and prof["values"], prof.keys()
    assert "window_years" in prof and "asof" in prof
    # 至少含闸门用到的一项判据，证明与筛选/闸门是同一份画像口径
    assert {"dv_yield", "roe_avg", "dividend_continuity_years",
            "payout_ratio", "fcf_dividend_cover"} & set(prof["values"])

    # 回测详情页的「成交个股的实时画像」表用的是**成交列表**接口，
    # 它也必须带 reason.profile（否则那张表会空）。
    from hdiv.web import service

    tl = service.list_backtest_trades(run_id, size=200)
    assert any((it.get("reason") or {}).get("profile") for it in tl["items"]), (
        "list_backtest_trades 未返回画像 —— 回测详情页的画像表会空"
    )


def test_cli_points_at_the_frontend() -> None:
    """一条命令跑完必须告诉用户「去前端哪里看」，而不是让用户再跑第二条命令。

    回归：曾把「看结果」写成需要手工传 run_id 去执行一个导出脚本 ——
    而 run_id 本来就在这条命令自己的输出里，前端也直接列出来。
    """
    import inspect

    from hdiv import cli

    src = inspect.getsource(cli)
    assert "_print_frontend_hint" in src
    assert "回测记录" in src, "提示里要给出前端位置"
    # daily 与 single 两条路径都必须给出该提示
    assert src.count('_print_frontend_hint(res["run_id"])') >= 2


class TestSpeedLevers:
    """速度旋钮：`--every-n-days` / `--signal-every-n-days`。

    两个频率**只改变「多久看一次」，不改变判定规则**。这里锁定三件事：
    参数真的传到了执行器、缓存键把重建频率算进去、耗时模型随频率正确下降。
    """

    def test_cli_flags_exist_and_parse(self) -> None:
        from hdiv.cli import build_parser

        p = build_parser()
        a = p.parse_args(["backtest"])
        assert a.every_n_days is None and a.signal_every_n_days is None, (
            "默认必须是 None（= 用 config 里的值，通常为 1）"
        )
        a = p.parse_args(["backtest", "--mode", "daily", "--start", "2020-01-05",
                          "--every-n-days", "5", "--signal-every-n-days", "10"])
        assert a.every_n_days == 5 and a.signal_every_n_days == 10

    def test_runner_accepts_overrides(self) -> None:
        from hdiv.backtest.daily import DailyRunner

        import inspect

        sig = inspect.signature(DailyRunner.run)
        assert "every_n_days" in sig.parameters
        assert "signal_every_n_days" in sig.parameters
        assert sig.parameters["every_n_days"].default is None
        assert sig.parameters["signal_every_n_days"].default is None

    def test_cache_key_depends_on_pool_cadence(self) -> None:
        """1 日 / 5 日 筛出的池子是不同的输入，不能共用缓存。"""
        from hdiv.backtest.daily import DailyRunner
        from hdiv.strategy.registry import StrategyRegistry

        reg = StrategyRegistry()
        s = reg.load("config/strategy/high_dividend_v1.yml")

        class _Stub:
            """只借 `_pools_cache_key`，不构造真 runner（那会连库）。"""

            strategy = s
            registry = reg
            _pools_cache_key = DailyRunner._pools_cache_key

        stub = _Stub()
        k1 = stub._pools_cache_key(date(2020, 1, 5), date(2026, 9, 30), 1)
        k5 = stub._pools_cache_key(date(2020, 1, 5), date(2026, 9, 30), 5)
        assert k1 != k5, "不同重建频率必须落在不同缓存文件"
        assert k1 == stub._pools_cache_key(date(2020, 1, 5), date(2026, 9, 30), 1), (
            "同样的输入必须得到同样的键（缓存要可复用）"
        )

    def test_time_model_shrinks_with_coarser_cadence(self) -> None:
        """耗时模型必须随频率下降（否则预计时长会骗人）。"""
        import hdiv.backtest.daily as D

        def sim_seconds(days: int, signal_step: int) -> float:
            return (days * D._SEC_PER_SIM_DAY_BASE
                    + days / signal_step * D._SEC_PER_SIM_SIGNAL)

        assert sim_seconds(1635, 1) > sim_seconds(1635, 5) > sim_seconds(1635, 21)
        # 逐日口径必须与实测同量级（43 日实测 131 秒）
        assert 100 < sim_seconds(43, 1) < 170, sim_seconds(43, 1)
        # 每 5 日口径也必须与实测同量级（约 28 秒）
        assert 15 < sim_seconds(43, 5) < 60, sim_seconds(43, 5)


# ---------------------------------------------------------------------------
# 性能优化的等价性（P1/P2/P4/P6）
# ---------------------------------------------------------------------------


def test_ttm_dps_series_prefix_equals_full() -> None:
    """P2 的前提：``ttm_dps_series`` 每一天的取值只依赖该日期与事件。

    若这条不成立，「整段算一次再切片」就会与「每次按前缀重算」不同 ——
    P2（消除 O(交易日数²)）正是建立在这条性质上。
    """
    from hdiv.factor.dividend_yield import ttm_dps_series

    idx = pd.bdate_range("2020-01-01", periods=800)
    # 事件间隔刻意覆盖三种情况：年内多次、略小于一年（重叠虚高）、略大于一年（断档虚低）
    ex = [idx[50], idx[300], idx[520], idx[640], idx[770]]
    ev = pd.DataFrame({
        "ex_date": ex,
        "imp_ann_date": [e - pd.Timedelta(days=12) for e in ex],
        "cash_div_tax": [0.5, 0.3, 0.6, 0.55, 0.7],
    })
    full = ttm_dps_series(idx, ev, ttm_days=365, grace_days=45, smooth_spikes=True)
    for i in (60, 310, 530, 650, 700, 799):
        pref = ttm_dps_series(idx[: i + 1], ev, ttm_days=365, grace_days=45,
                              smooth_spikes=True)
        assert abs(float(full[i]) - float(pref[-1])) < 1e-12, (
            f"第 {i} 天：整段 {full[i]} != 前缀末值 {pref[-1]} —— P2 的前提不成立"
        )


@pytest.mark.db
def test_pit_dividend_scope_is_exact_subset() -> None:
    """P4：``restrict_to`` 只能把 ``dividend_records`` 收窄成精确子集。"""
    asof = date(2024, 6, 3)
    pit = _load_pit(date(2024, 1, 1), date(2024, 6, 28), ref_end=date(2024, 12, 31))
    full = pit.dividend_records(asof, years_back=8)
    assert not full.empty
    syms = sorted(full["symbol"].astype(str).unique())[:5]
    pit.restrict_to(syms)
    try:
        scoped = pit.dividend_records(asof, years_back=8)
    finally:
        pit.restrict_to(None)
    cols = list(full.columns)
    want = full[full["symbol"].isin(set(syms))].reset_index(drop=True)
    pd.testing.assert_frame_equal(
        scoped.sort_values(cols, ignore_index=True),
        want.sort_values(cols, ignore_index=True),
        check_dtype=False, rtol=1e-9,
    )


@pytest.mark.db
def test_pit_dividend_events_matches_direct_repo(plain_repo) -> None:
    """P6：内存版 ``dividend_events`` 必须与直连逐值一致（含 NULL 公告日边界）。"""


    ref_end = date(2026, 9, 30)
    pit = _load_pit(date(2025, 1, 1), ref_end, ref_end=ref_end)
    for s, e in ((date(2024, 1, 1), date(2024, 12, 31)),
                 (date(2020, 1, 1), ref_end)):
        a = pit.dividend_events(s, e)
        b = plain_repo.dividend_events(s, e)
        cols = list(a.columns)
        pd.testing.assert_frame_equal(
            a.sort_values(cols, ignore_index=True),
            b.sort_values(cols, ignore_index=True),
            check_dtype=False, rtol=1e-9,
            obj=f"dividend_events {s}..{e}",
        )
    # 超出参照终点的区间必须退回直连，而不是返回少行的表
    narrow = _load_pit(date(2025, 1, 1), date(2024, 12, 31),
                       ref_end=date(2024, 6, 3))
    a = narrow.dividend_events(date(2024, 1, 1), date(2024, 12, 31))
    b = plain_repo.dividend_events(date(2024, 1, 1), date(2024, 12, 31))
    assert len(a) == len(b), "超出参照终点的区间被静默截断了"


@pytest.mark.db
def test_profile_prepare_accepts_external_price(plain_repo) -> None:
    """P6：引擎把已取好的价格交给画像复用，结果必须逐值不变。"""
    from hdiv.profile.pit import PitProfileService

    syms = ["600036.SH", "000651.SZ", "601398.SH"]
    a, b = date(2015, 1, 1), date(2026, 9, 30)
    pit = _load_pit(a, b, ref_end=b)
    s1 = PitProfileService(window_years=5, repo=pit)
    s1.prepare(syms, a, b)
    price = plain_repo.price_history(syms, a, b, adjust="none")
    s2 = PitProfileService(window_years=5, repo=pit)
    s2.prepare(syms, a, b, price=price)
    s1.configure({"dv_yield", "pe_ttm", "pb", "roe_avg"})
    s2.configure({"dv_yield", "pe_ttm", "pb", "roe_avg"})
    for d in (date(2018, 5, 18), date(2024, 6, 3)):
        for y in syms:
            x, z = s1.snapshot(y, d), s2.snapshot(y, d)
            assert (x is None) == (z is None)
            if x is not None and z is not None:
                assert x.values == z.values and x.percentiles == z.percentiles


@pytest.mark.db
def test_dividend_scope_does_not_change_selection(pit_2024) -> None:
    """P4 的端到端保证：收窄分红取数不改变每日入选成员。"""
    from hdiv.universe.pit import PitRepo
    from hdiv.universe.selector import UniverseSelector

    pit = pit_2024
    cfg = load_config("universe")
    sel = UniverseSelector(cfg, repo=pit)
    days = [d for d in pit.trading_days(date(2024, 1, 1), date(2024, 6, 28))
            if d in (date(2024, 1, 2), date(2024, 3, 1), date(2024, 6, 3))]
    assert days

    orig = PitRepo.dividend_records

    def _unscoped(self, asof, **kw):  # noqa: ANN001
        saved = self._scope
        self._scope = None
        try:
            return orig(self, asof, **kw)
        finally:
            self._scope = saved

    # 关掉收窄（= 优化前的行为）
    PitRepo.dividend_records = _unscoped
    try:
        before = {d: set(sel.run(asof=d, persist=False, verbose=False)["selected"]["symbol"])
                  for d in days}
    finally:
        PitRepo.dividend_records = orig
    # 打开收窄：走生产路径（on_stage 钩子会按阶段收窄）
    pit.set_candidate_scope(None)
    screener = DailyUniverseScreener(cfg, pit, verbose=False)
    after = {d: set(screener.screen_day(d).symbols) for d in days}
    for d in days:
        assert before[d] == after[d], (
            f"{d} 收窄分红取数改变了入选：多 {sorted(after[d] - before[d])} "
            f"少 {sorted(before[d] - after[d])}"
        )


class TestPoolExitSemantics:
    """动态池的语义核心：掉出当日池子的持仓怎么办。

    用合成行情直接调 ``_evaluate``，不经数据库 —— 这样 hold / sell 两条分支
    都被真正执行到，而不是只靠「源码里有这个词」。
    """

    @staticmethod
    def _setup(pool_exit_action: str):
        import pandas as pd

        from hdiv.backtest.engine import BacktestEngine, Position
        from hdiv.core.config import load_config as _lc
        from hdiv.strategy.registry import StrategyRegistry

        reg = StrategyRegistry()
        s = reg.load("config/strategy/high_dividend_v1.yml").model_copy(deep=True)
        # 闸门关闭：本用例考的是**池子语义**，不是画像闸门。
        s.entry.profile_gate.enabled = False
        bt = _lc("backtest")
        eng = BacktestEngine(
            s, backtest=bt, pool_exit_action=pool_exit_action,
            signal_frequency_days=1,
        )

        # 400 个交易日的合成行情：收盘价从 40 线性跌到 10 → 股息率一路上行，
        # 当前值必然是窗口内的最高分位（≈100%），足以触发最高买入档。
        idx = pd.bdate_range(end="2024-06-28", periods=400)
        close = np.linspace(40.0, 10.0, len(idx))
        px = pd.DataFrame({"open": close, "close": close}, index=idx)
        day = idx[-1].date()
        ex = idx[200]
        events = {"X.SH": pd.DataFrame([{
            "ex_date": ex, "imp_ann_date": ex - pd.Timedelta(days=10),
            "cash_div_tax": 1.0,
        }])}
        px_by_sym = {"X.SH": px}
        ctx = {"px_by_sym": px_by_sym, "events": events,
               "yield_by_sym": build_yield_series(px_by_sym, events)}
        pos = {"X.SH": Position(symbol="X.SH", quantity=1000.0, avg_cost=20.0,
                                first_buy_date=day - timedelta(days=100),
                                last_buy_date=day - timedelta(days=100),
                                cost_basis=20000.0)}
        return eng, day, ctx, pos

    def test_high_percentile_triggers_buy_when_not_held(self) -> None:
        eng, day, ctx, _ = self._setup("hold")
        sigs = eng._evaluate(day, 1e6, {}, {"X.SH"}, ctx)
        assert [x.kind for x in sigs] == ["BUY"], sigs
        assert sigs[0].reason["in_universe"] is True

    def test_in_pool_holding_can_add(self) -> None:
        eng, day, ctx, pos = self._setup("hold")
        sigs = eng._evaluate(day, 1e6, pos, {"X.SH"}, ctx)
        assert [x.kind for x in sigs] == ["ADD"], sigs

    def test_out_of_pool_holding_blocks_add_but_not_liquidate(self) -> None:
        """``hold``：掉出池子 → 停止加仓，但**不清仓**（留 HOLD 记录可追溯）。"""
        eng, day, ctx, pos = self._setup("hold")
        sigs = eng._evaluate(day, 1e6, pos, set(), ctx)
        assert [x.kind for x in sigs] == ["HOLD"], sigs
        assert sigs[0].reason["skip_reason"] == "OUT_OF_UNIVERSE"
        assert sigs[0].reason["executed"] is False
        assert sigs[0].reason["in_universe"] is False
        # 不进入待成交队列 ⇒ 不会被 _execute 清算
        assert sigs[0].kind not in {"BUY", "ADD", "SELL", "TRIM"}
        assert pos["X.SH"].quantity == 1000.0

    def test_out_of_pool_holding_liquidates_when_configured(self) -> None:
        """``sell``：掉出池子即清仓（可选项，不是默认）。"""
        eng, day, ctx, pos = self._setup("sell")
        sigs = eng._evaluate(day, 1e6, pos, set(), ctx)
        assert [x.kind for x in sigs] == ["SELL"], sigs
        assert sigs[0].target_weight == 0.0

    def test_pool_exit_sell_branch_is_exercised_by_both_actions(self) -> None:
        """两种配置必须给出**不同**的动作，否则 pool_exit_action 是死配置。"""
        a, day, ctx, pos_a = self._setup("hold")
        b, _, ctx_b, pos_b = self._setup("sell")
        ka = [x.kind for x in a._evaluate(day, 1e6, pos_a, set(), ctx)]
        kb = [x.kind for x in b._evaluate(day, 1e6, pos_b, set(), ctx_b)]
        assert ka != kb, f"pool_exit_action 未生效：{ka} == {kb}"


class TestMemberRows:
    def _screen(self, symbols: list[str]) -> object:
        from hdiv.universe.daily import ScreenDay

        m = pd.DataFrame({
            "symbol": symbols,
            "name": ["A", "B", "C"][: len(symbols)],
            "industry": ["银行"] * len(symbols),
            "dividend_yield": [0.06, 0.05, 0.04][: len(symbols)],
            "total_mv": [1e11, 2e11, 3e11][: len(symbols)],
            "roe_avg": [0.12, 0.11, 0.10][: len(symbols)],
        })
        return ScreenDay(
            trade_date=date(2024, 3, 1),
            candidate_count=5000,
            member_count=len(symbols),
            symbols=symbols,
            members=m,
            stats={"market": 10},
        )

    def test_rows_shape(self) -> None:
        from datetime import datetime

        from hdiv.universe.daily import DailyUniverseScreener

        rows = DailyUniverseScreener.member_rows(
            "run1", [self._screen(["600036.SH", "601398.SH"])],
            created_at=datetime(2024, 3, 1, 15, 0, 0),
        )
        assert len(rows) == 2
        r = rows[0]
        assert r["run_id"] == "run1"
        assert r["trade_date"] == date(2024, 3, 1)
        assert r["symbol"] == "600036.SH"
        assert r["candidate_count"] == 5000
        assert r["dividend_yield"] == pytest.approx(0.06)
        # values_json 必须是可解析的 JSON（供「为什么是这只」复核）
        import json

        v = json.loads(r["values_json"])
        assert v["dividend_yield"] == pytest.approx(0.06)
        assert v["roe_avg"] == pytest.approx(0.12)

    def test_empty_members_yield_no_rows(self) -> None:
        from datetime import datetime

        from hdiv.universe.daily import DailyUniverseScreener, ScreenDay

        s = ScreenDay(date(2024, 3, 1), 5000, 0, [], pd.DataFrame())
        assert DailyUniverseScreener.member_rows(
            "r", [s], created_at=datetime(2024, 3, 1)
        ) == []


# ---------------------------------------------------------------------------
# DB：PitRepo 与 Repo 逐值一致
# ---------------------------------------------------------------------------


def _load_pit(start: date, end: date, ref_end: date | None = None):
    """载入一个 PitRepo（库不可用时 skip）。

    **载入很贵**（参照数据约 17 秒 + 区块约 25 秒），因此这些用例共享
    module 级 fixture，而不是各自新建一个 —— 否则整个文件的耗时是分钟级的倍数。
    """
    from hdiv.universe.pit import PitRepo

    try:
        pit = PitRepo()
        pit.load_reference(end=ref_end or end)
        pit.load_range(start, end)
    except Exception as exc:  # pragma: no cover - 环境相关
        pytest.skip(f"数据库不可用：{exc}")
    return pit


#: 等价性 / 剪枝用例共用的区间（覆盖 2024 上半年，含年报季）
_RANGE_START, _RANGE_END = date(2024, 1, 1), date(2024, 6, 28)
_ASOF = date(2024, 6, 3)


@pytest.fixture(scope="module")
def pit_2024():
    """共享的 PitRepo（2024 上半年）。用例只读，唯一例外是剪枝用例会改候选范围。"""
    from hdiv.universe.pit import PitRepo

    try:
        pit = PitRepo()
        pit.load_reference(end=date(2024, 12, 31))
        pit.load_range(_RANGE_START, _RANGE_END)
    except Exception as exc:  # pragma: no cover
        pytest.skip(f"数据库不可用：{exc}")
    yield pit
    pit.release_range()


@pytest.fixture(scope="module")
def plain_repo():
    from hdiv.data.repo import Repo

    try:
        repo = Repo()
        repo.trading_day(_ASOF)  # 探活
    except Exception as exc:  # pragma: no cover
        pytest.skip(f"数据库不可用：{exc}")
    return repo


def _assert_same_multiset(name: str, a: pd.DataFrame, b: pd.DataFrame) -> None:
    """按**全列排序**后逐值比较（行序不同不代表内容不同）。

    用全部列排序而不是挑几列：财务表里同一 (symbol, 财年) 可能有多条重述公告，
    只按业务键排序会让等值行两两错配，产生假报警。
    """
    assert list(a.columns) == list(b.columns), (
        f"{name} 列集合不一致：{list(a.columns)} vs {list(b.columns)}"
    )
    if a.empty and b.empty:
        return
    cols = list(a.columns)
    x = a.sort_values(cols, ignore_index=True)
    y = b.sort_values(cols, ignore_index=True)
    assert x.shape == y.shape, f"{name} 行数不一致：{x.shape} vs {y.shape}"
    pd.testing.assert_frame_equal(
        x, y, check_dtype=False, rtol=1e-9, atol=1e-12,
        obj=f"{name} 与直连口径不一致",
    )


@pytest.mark.db
def test_pit_repo_matches_direct_repo(pit_2024, plain_repo) -> None:
    """PitRepo 的每一个被覆盖的方法都必须与直连 Repo 逐值一致。"""
    asof = _ASOF
    pit, plain = pit_2024, plain_repo

    _assert_same_multiset(
        "market_panel(lookback=5)",
        pit.market_panel(asof, lookback_days=5),
        plain.market_panel(asof, lookback_days=5),
    )
    _assert_same_multiset(
        "market_panel(lookback=0)",
        pit.market_panel(asof),
        plain.market_panel(asof),
    )
    _assert_same_multiset(
        "avg_amount(20)",
        pit.avg_amount(asof, window=20),
        plain.avg_amount(asof, window=20),
    )
    syms = ["600036.SH", "000651.SZ"]
    _assert_same_multiset(
        "avg_amount(20, symbols)",
        pit.avg_amount(asof, window=20, symbols=syms),
        plain.avg_amount(asof, window=20, symbols=syms),
    )
    _assert_same_multiset(
        "financial_panel",
        pit.financial_panel(asof),
        plain.financial_panel(asof),
    )
    _assert_same_multiset(
        "annual_financial_history(5)",
        pit.annual_financial_history(asof, years=5),
        plain.annual_financial_history(asof, years=5),
    )
    _assert_same_multiset(
        "annual_financial_history(11)",
        pit.annual_financial_history(asof, years=11),
        plain.annual_financial_history(asof, years=11),
    )
    _assert_same_multiset(
        "annual_financial_averages(5)",
        pit.annual_financial_averages(asof, years=5),
        plain.annual_financial_averages(asof, years=5),
    )
    _assert_same_multiset(
        "annual_financials(10)",
        pit.annual_financials(asof, years=10),
        plain.annual_financials(asof, years=10),
    )
    _assert_same_multiset(
        "dividend_records(8)",
        pit.dividend_records(asof, years_back=8),
        plain.dividend_records(asof, years_back=8),
    )
    _assert_same_multiset(
        "dividend_records(all)",
        pit.dividend_records(asof, years_back=8, implemented_only=False),
        plain.dividend_records(asof, years_back=8, implemented_only=False),
    )
    assert pit.trading_day(asof) == plain.trading_day(asof)
    assert pit.prev_trading_day(asof) == plain.prev_trading_day(asof)
    assert len(pit.trading_days(date(2024, 1, 1), asof)) == len(
        plain.trading_days(date(2024, 1, 1), asof)
    )
    assert pit.suspended_on(asof) == plain.suspended_on(asof)
    assert pit.st_symbols(asof) == plain.st_symbols(asof)


@pytest.mark.db
def test_pit_repo_symbols_subset_is_exact(pit_2024, plain_repo) -> None:
    """``symbols`` 过滤只是全市场结果取子集：两者必须一致（含缓存命中路径）。"""
    asof, pit, plain = _ASOF, pit_2024, plain_repo
    syms = ["600036.SH", "601398.SH", "000651.SZ"]

    full = pit.financial_panel(asof)
    sub = pit.financial_panel(asof, symbols=syms)
    _assert_same_multiset(
        "financial_panel(symbols) 子集",
        sub, full[full["symbol"].isin(syms)].reset_index(drop=True),
    )
    _assert_same_multiset(
        "financial_panel(symbols) vs 直连",
        sub, plain.financial_panel(asof, symbols=syms),
    )


@pytest.mark.db
def test_pit_repo_caches_are_exact() -> None:
    """可见性缓存必须**精确**：命中与不命中给出同一结果。

    缓存键是「已公告财报条数」。这里把缓存清空强制重算一次，与命中结果比对 ——
    若不相等，说明键不是充分统计量（那就会静默用错的面板做决策）。
    """
    asof = date(2024, 4, 25)  # 年报季，可见集合天天变
    pit = _load_pit(date(2024, 3, 1), date(2024, 6, 28), ref_end=date(2024, 12, 31))
    hit = pit.financial_panel(asof)
    pit._memo.clear()
    cold = pit.financial_panel(asof)
    _assert_same_multiset("financial_panel 缓存命中 vs 冷算", hit, cold)

    h1 = pit.annual_financial_history(asof, years=11)
    pit._memo.clear()
    h2 = pit.annual_financial_history(asof, years=11)
    _assert_same_multiset("annual_financial_history 命中 vs 冷算", h1, h2)


@pytest.mark.db
def test_pit_repo_out_of_range_raises() -> None:
    """区间外必须**报错**，不能返回空表 —— 空表会被当成「当天没有股票」。"""
    from hdiv.core.errors import HdivError

    pit = _load_pit(date(2024, 5, 1), date(2024, 6, 28))
    with pytest.raises(HdivError):
        pit.market_panel(date(2019, 1, 2), lookback_days=5)
    with pytest.raises(HdivError):
        pit.avg_amount(date(2019, 1, 2), window=20)  # 全市场查询必须报错


@pytest.mark.db
def test_pit_repo_avg_amount_falls_back_for_narrow_query(plain_repo) -> None:
    """区块释放后，**带 symbol 的**均额查询仍可回答（画像路径依赖它）。"""
    pit = _load_pit(date(2024, 5, 1), date(2024, 6, 28))
    asof = date(2024, 6, 3)
    pit.release_range()
    got = pit.avg_amount(asof, window=20, symbols=["600036.SH"])
    want = plain_repo.avg_amount(asof, window=20, symbols=["600036.SH"])
    _assert_same_multiset("avg_amount 窄查询回退", got, want)


# ---------------------------------------------------------------------------
# DB：预剪枝等价性
# ---------------------------------------------------------------------------


@pytest.mark.db
def test_prune_does_not_change_selection(pit_2024) -> None:
    """开/关预剪枝必须选出完全相同的成员（否则它是近似，不是等价）。"""
    from hdiv.universe.selector import UniverseSelector

    pit = pit_2024
    start, end = _RANGE_START, _RANGE_END
    cfg = load_config("universe")
    sel = UniverseSelector(cfg, repo=pit)

    # 几天样本：月初、季报期、年报季、月末
    wanted = {date(2024, 1, 2), date(2024, 3, 1), date(2024, 4, 25),
              date(2024, 6, 3), date(2024, 6, 28)}
    days = [d for d in pit.trading_days(start, end) if d in wanted]
    assert days, "样本交易日为空"

    # 先不剪枝
    pit.set_candidate_scope(None)
    base = {d: set(sel.run(asof=d, persist=False, verbose=False)["selected"]["symbol"])
            for d in days}

    # 再剪枝
    DailyUniverseScreener(cfg, pit, verbose=False).build_prune_set(start, end)
    pruned = {d: set(sel.run(asof=d, persist=False, verbose=False)["selected"]["symbol"])
              for d in days}
    pit.set_candidate_scope(None)  # 复位，避免影响其它用例

    for d in days:
        assert base[d] == pruned[d], (
            f"{d} 预剪枝改变了最终入选："
            f"多出 {sorted(pruned[d] - base[d])}，丢失 {sorted(base[d] - pruned[d])}"
        )


@pytest.mark.db
def test_prune_set_keeps_every_actual_member(pit_2024) -> None:
    """预剪枝的保留集合必须**包含**每天实际选出的成员（上界论证的实证）。"""
    from hdiv.universe.selector import UniverseSelector

    pit = pit_2024
    start, end = _RANGE_START, _RANGE_END
    cfg = load_config("universe")
    allowed = DailyUniverseScreener(cfg, pit, verbose=False).build_prune_set(start, end)
    pit.set_candidate_scope(None)  # 不剪枝地真筛一次
    sel = UniverseSelector(cfg, repo=pit)
    for d in [date(2024, 3, 1), date(2024, 6, 3)]:
        got = set(sel.run(asof=d, persist=False, verbose=False)["selected"]["symbol"])
        assert got <= allowed, f"{d} 有成员被预剪枝误剔：{sorted(got - allowed)}"


# ---------------------------------------------------------------------------
# DB：PIT 纪律 —— 股票池不随「回测终点」变化
# ---------------------------------------------------------------------------


@pytest.mark.db
def test_daily_pool_is_point_in_time() -> None:
    """某一天的股票池只依赖该日及之前的数据，与回测终点无关。

    这是「无未来函数」在每日选股上的直接检验：把参照数据的终点推后一年，
    同一天的选股结果必须**逐只相同**。
    """
    from hdiv.universe.selector import UniverseSelector

    asof = date(2023, 6, 1)
    # 只载入一个月，控制成本（这条检验比的是「两个终点是否给出同一答案」）
    a = _load_pit(date(2023, 5, 1), asof, ref_end=asof)
    b = _load_pit(date(2023, 5, 1), asof, ref_end=asof + timedelta(days=365))
    cfg = load_config("universe")
    ra = UniverseSelector(cfg, repo=a).run(asof=asof, persist=False, verbose=False)
    rb = UniverseSelector(cfg, repo=b).run(asof=asof, persist=False, verbose=False)
    assert set(ra["selected"]["symbol"]) == set(rb["selected"]["symbol"])


# ---------------------------------------------------------------------------
# DB：端到端（短区间）
# ---------------------------------------------------------------------------

# 回测 run 的从属表（顺序 = 删除顺序：先子后父）
_BACKTEST_CHILD_TABLES = (
    "hd_daily_universe",
    "hd_backtest_equity",
    "hd_backtest_metric",
    "hd_backtest_position",
    "hd_backtest_signal",
    "hd_backtest_trade",
)


def _purge_daily_run(run_id: str) -> None:
    """物理删除本次端到端测试自己写入的 run 及其从属行。

    **测试不得在分析库里留垃圾**：每跑一次都会多出一条同名回测记录（run_id 指纹
    含 ``datetime.now()``，见 ``hdiv.backtest.engine``），跑几十次后前端「回测
    记录」就被测试产物淹掉，而它对外看起来和真实回测没有区别。

    项目的 ``StatementGuard`` 有意禁止 DELETE（只增不删），所以这里自建一个
    **不装守卫**的连接，并严格按 ``run_id`` 精确回收刚刚写入的行。
    """
    from sqlalchemy import create_engine, text

    from hdiv.data import db

    cfg = load_config("datasource")
    db.load_dotenv_once()
    engine = create_engine(db.build_url(cfg), pool_pre_ping=True, future=True)
    try:
        with engine.begin() as conn:
            for table in (*_BACKTEST_CHILD_TABLES, "hd_backtest_run"):
                conn.execute(text(f"DELETE FROM {table} WHERE run_id = :r"),
                             {"r": run_id})
    finally:
        engine.dispose()


@pytest.mark.db
def test_daily_run_persists_members_and_run() -> None:
    """daily 端到端：落库 run（mode=daily）与每日选股成员（跑完自清理）。"""
    from hdiv.backtest.daily import DailyRunner
    from hdiv.data import db

    try:
        runner = DailyRunner.from_strategy("config/strategy/high_dividend_v1.yml")
    except Exception as exc:  # pragma: no cover
        pytest.skip(f"数据库不可用：{exc}")
    start, end = date(2024, 3, 1), date(2024, 3, 15)
    try:
        res = runner.run(start=start, end=end, persist=True, verbose=False)
    except Exception as exc:  # pragma: no cover
        pytest.skip(f"数据不足，跳过端到端：{exc}")

    run_id = res["run_id"]
    try:
        assert res["mode"] == "daily"
        cfg = load_config("datasource")
        assert db.table_exists("hd_daily_universe", cfg)
        got = db.read_sql(
            "SELECT COUNT(*) AS n, COUNT(DISTINCT trade_date) AS d "
            "FROM hd_daily_universe WHERE run_id = :r",
            {"r": run_id}, cfg=cfg,
        )
        assert int(got["n"].iloc[0]) > 0, "未落库任何每日选股成员"
        assert int(got["d"].iloc[0]) >= 5, "落库的决策时点太少"
        run = db.read_sql(
            "SELECT mode, universe_run_id FROM hd_backtest_run WHERE run_id = :r",
            {"r": run_id}, cfg=cfg,
        )
        assert len(run) == 1
        assert run["mode"].iloc[0] == "daily"
        # daily 是动态池：不得关联任何冻结股票池
        assert pd.isna(run["universe_run_id"].iloc[0]) or run["universe_run_id"].iloc[0] is None
    finally:
        # 无论断言成功还是失败，都不能把测试产物留在分析库里
        _purge_daily_run(run_id)