ggx/tests/test_dividend_smoothing.py

"""TTM 股息率毛刺消除测试。

**背景**：A 股相邻两次除权的间隔经常不是 365 天。硬 365 天窗口于是在每年
除权日附近制造两种日历假象：

- **重叠虚高**：间隔 < 365 时新旧分红同时在窗口内。实测招商银行 2015-07-03
  股息率 0.620 → 1.290（+108%），10 天后回落到 0.670。
- **断档虚低**：间隔 > 365 时旧的已到期而新的未入场。实测中国神华
  2016-07-04：0.740 → 0.320（−57%）。

两者都会污染「历史分位」这一核心信号，且筛选器用的也是同一个数
（直接决定选股），因此必须消除。
"""

from __future__ import annotations

import numpy as np
import pandas as pd
import pytest

from hdiv.factor.dividend_yield import (
    build_dps_events,
    dedupe_dividend_events,
    ttm_dps_at,
    ttm_dps_series,
    ttm_params,
)

TTM = 365
GRACE = 45


def _events(pairs: list[tuple[str, float]]) -> pd.DataFrame:
    """构造分红事件表：(除权日, 金额)。"""
    return pd.DataFrame({
        "ex_date": pd.to_datetime([d for d, _ in pairs]),
        "imp_ann_date": pd.to_datetime([d for d, _ in pairs]),
        "cash_div_tax": [a for _, a in pairs],
    })


def _daily(start: str, end: str) -> pd.DatetimeIndex:
    return pd.date_range(start, end, freq="D")


# ---------------------------------------------------------------------------
# 核心：两种毛刺都要消除
# ---------------------------------------------------------------------------


def test_overlap_spike_is_removed() -> None:
    """间隔 360 天：新分红入场时旧的不应再计入（消除 +100% 虚高）。"""
    d = _daily("2020-01-01", "2023-12-31")
    ev = _events([("2021-06-01", 1.0), ("2022-05-27", 1.2), ("2023-05-22", 1.4)])

    raw = pd.Series(ttm_dps_series(d, ev, ttm_days=TTM, grace_days=GRACE,
                                   smooth_spikes=False), index=d)
    sm = pd.Series(ttm_dps_series(d, ev, ttm_days=TTM, grace_days=GRACE,
                                  smooth_spikes=True), index=d)

    # 未平滑时：2022-05-27 当刻涨到 1.0+1.2=2.2，5 天后旧的到期回落
    assert raw.max() > 2.1, f"未平滑应出现重叠虚高，实际 max={raw.max()}"
    # 平滑后：不应出现两笔相加
    assert sm.max() <= 1.45, f"平滑后不应双算，实际 max={sm.max()}"
    # 且切换当天不跳变
    after = sm.loc[pd.Timestamp("2022-05-27"):].iloc[:5]
    assert after.max() / after.min() - 1 < 0.05, "接管当天不应有跳变"


def test_gap_dip_is_filled() -> None:
    """间隔 370 天：旧的到期后应继续计到新的入场（消除断档虚低）。"""
    d = _daily("2020-01-01", "2023-12-31")
    ev = _events([("2021-06-01", 1.0), ("2022-06-06", 1.2)])   # 间隔 370 天

    raw = pd.Series(ttm_dps_series(d, ev, ttm_days=TTM, grace_days=GRACE,
                                   smooth_spikes=False), index=d)
    sm = pd.Series(ttm_dps_series(d, ev, ttm_days=TTM, grace_days=GRACE,
                                  smooth_spikes=True), index=d)

    # 只看「第一笔到第二笔」这段（尾部无后继本就该归零，属正确行为）
    win = slice(pd.Timestamp("2021-06-01"), pd.Timestamp("2022-06-06"))
    raw_a, sm_a = raw.loc[win], sm.loc[win]
    # 未平滑：2022-06-01 旧的到期、新的还没来 → 归零 5 天
    assert raw_a.min() == 0.0, "未平滑应出现断档归零"
    # 平滑后：同区间不应归零
    assert sm_a.min() > 0.9, f"平滑后不应断档，实际 min={sm_a.min()}"


def test_intra_year_multiple_payments_are_not_merged() -> None:
    """年内多次分红（间隔 180 天）必须都保留 —— 否则会把中期分红误删。"""
    d = _daily("2021-01-01", "2023-12-31")
    ev = _events([
        ("2022-06-01", 0.3), ("2022-11-28", 0.7),
        ("2023-05-29", 0.3), ("2023-11-25", 0.7),
    ])
    sm = pd.Series(ttm_dps_series(d, ev, ttm_days=TTM, grace_days=GRACE,
                                  smooth_spikes=True), index=d)
    # 年中确实应同时含两笔（0.3 + 0.7 = 1.0）。
    # 重叠期 = 第二笔入场（2022-11-28）到第一笔满 365 天（2023-05-28），
    # 区间右端取半开语义：2023-05-29 当天第一笔已经到期，只剩 0.3。
    peak = sm.loc[pd.Timestamp("2022-11-28"):pd.Timestamp("2023-05-28")]
    assert peak.max() > 0.95, f"年内两笔分红应同时计入，实际 max={peak.max()}"
    assert sm.loc[pd.Timestamp("2022-11-28")] == pytest.approx(1.0)


def test_true_cessation_still_goes_to_zero() -> None:
    """真停发必须如实归零，不能因为平滑就永远挂着旧分红。"""
    d = _daily("2020-01-01", "2025-12-31")
    ev = _events([("2021-06-01", 1.0), ("2023-06-01", 1.0)])   # 中间空了两年
    sm = pd.Series(ttm_dps_series(d, ev, ttm_days=TTM, grace_days=GRACE,
                                  smooth_spikes=True), index=d)
    # 2021 那笔在 2022-06-01 + 45 天宽限后必须归零
    gap = sm.loc[pd.Timestamp("2022-09-01"):pd.Timestamp("2023-05-31")]
    assert gap.max() == 0.0, f"停发期间应归零，实际 max={gap.max()}"


def test_smoothing_can_be_disabled() -> None:
    """smooth_spikes=False 应精确复现旧的「硬窗口 + 归零才兜底」行为。"""
    d = _daily("2020-01-01", "2023-12-31")
    ev = _events([("2021-06-01", 1.0), ("2022-05-27", 1.2)])
    off = ttm_dps_series(d, ev, ttm_days=TTM, grace_days=GRACE, smooth_spikes=False)
    # 旧行为：重叠期双算
    assert off.max() >= 2.1
    on = ttm_dps_series(d, ev, ttm_days=TTM, grace_days=GRACE, smooth_spikes=True)
    assert on.max() < off.max()


def test_build_dps_events_matches_series_expectations() -> None:
    """事件表经 build_dps_events 规范化后仍可用。"""
    raw = pd.DataFrame({
        "symbol": ["X"] * 3,
        "ex_date": ["2021-06-01", "2022-05-27", "2023-05-22"],
        "imp_ann_date": ["2021-05-25", "2022-05-20", "2023-05-15"],
        "cash_div_tax": [1.0, 1.2, 1.4],
    })
    e = build_dps_events(raw)["X"]
    d = _daily("2021-01-01", "2023-12-31")
    out = ttm_dps_series(d, e, ttm_days=TTM, grace_days=GRACE, smooth_spikes=True)
    assert len(out) == len(d) and out.max() <= 1.45


# ---------------------------------------------------------------------------
# 经济事件口径：同一 (symbol, ex_date) 的多条记录只能算一笔
# ---------------------------------------------------------------------------


def test_dedupe_collapses_duplicate_announcements() -> None:
    """同一除权日的重复记录（含「分项 + 合计」）必须收敛成一笔。"""
    raw = pd.DataFrame({
        "symbol": ["X"] * 3 + ["Y"],
        "ex_date": ["2024-11-07"] * 3 + ["2024-05-10"],
        "imp_ann_date": ["2024-08-29", "2024-10-11", "2024-10-30", "2024-05-06"],
        "cash_div_tax": [0.4, 1.0, 1.4, 0.5],
    })
    out = dedupe_dividend_events(raw)
    assert len(out) == 2
    x = out[out["symbol"] == "X"].iloc[0]
    # 实测 002352.SZ 2024-11-07：0.4 + 1.0 = 1.4，取合计口径
    assert float(x["cash_div_tax"]) == pytest.approx(1.4)
    # PIT：取**最晚**公告日，绝不早于该金额真正公告的时间
    assert pd.Timestamp(x["imp_ann_date"]) == pd.Timestamp("2024-10-30")
    assert len(out[out["symbol"] == "Y"]) == 1


def test_dedupe_keeps_stock_dividend_alongside_cash() -> None:
    """一行纯现金 + 一行纯送转时不得丢掉送转（逐字段取最大，而非整行取最大）。"""
    raw = pd.DataFrame({
        "symbol": ["X", "X"],
        "ex_date": ["2024-07-01", "2024-07-01"],
        "imp_ann_date": ["2024-06-01", "2024-06-20"],
        "cash_div_tax": [0.5, None],
        "stk_div": [None, 0.3],
        "stk_bo_rate": [None, 0.3],
    })
    out = dedupe_dividend_events(raw)
    assert len(out) == 1
    assert float(out.iloc[0]["cash_div_tax"]) == pytest.approx(0.5)
    assert float(out.iloc[0]["stk_div"]) == pytest.approx(0.3)


def test_dedupe_is_identity_without_duplicates() -> None:
    """没有重复时原样返回（热路径短路，不得改变 dtype 或行序）。"""
    raw = pd.DataFrame({
        "symbol": ["X", "X"],
        "ex_date": ["2021-06-01", "2022-05-27"],
        "cash_div_tax": [1.0, 1.2],
    })
    out = dedupe_dividend_events(raw)
    assert out is raw


def test_ttm_series_dedupes_by_default() -> None:
    """TTM 入口默认按经济事件聚合；重复记录不得把股息率抬高。"""
    dup = pd.DataFrame({
        "ex_date": pd.to_datetime(["2022-06-01", "2022-06-01"]),
        "imp_ann_date": pd.to_datetime(["2022-05-20", "2022-05-25"]),
        "cash_div_tax": [0.5, 0.5],
    })
    d = _daily("2022-01-01", "2023-12-31")
    on = ttm_dps_series(d, dup, ttm_days=TTM, grace_days=GRACE)
    assert on.max() == pytest.approx(0.5)
    # 关掉聚合应复现**旧行为**：同一天两条被当成两笔独立分红累加（1.0）。
    # 这正是本 bug 的来源 —— 业务路径必须保持默认聚合。
    off = ttm_dps_series(d, dup, ttm_days=TTM, grace_days=GRACE, dedupe_events=False)
    assert off.max() == pytest.approx(1.0), "关掉聚合应复现重复累加"


def test_partial_and_total_records_are_merged_not_summed() -> None:
    """「分项 + 合计」同一天多条：必须取合计（1.4），而不是累加成 2.8。"""
    raw = pd.DataFrame({
        "ex_date": pd.to_datetime(["2024-11-07"] * 3),
        "imp_ann_date": pd.to_datetime(["2024-08-29", "2024-10-11", "2024-10-30"]),
        "cash_div_tax": [0.4, 1.0, 1.4],
    })
    d = _daily("2024-01-01", "2025-12-31")
    on = ttm_dps_series(d, raw, ttm_days=TTM, grace_days=GRACE)
    assert on.max() == pytest.approx(1.4)


def test_ttm_series_survives_null_cash_on_stock_dividend_rows() -> None:
    """送转行的 cash_div_tax 是 NULL：不得让 NaN 传染整条 TTM 序列。"""
    ev = pd.DataFrame({
        "ex_date": pd.to_datetime(["2022-06-01", "2023-06-05"]),
        "imp_ann_date": pd.to_datetime(["2022-05-20", "2023-05-25"]),
        "cash_div_tax": [None, 0.8],
    })
    d = _daily("2023-01-01", "2023-12-31")
    out = ttm_dps_series(d, ev, ttm_days=TTM, grace_days=GRACE)
    assert np.isfinite(out).all(), "NULL 现金分红把 TTM 序列变成了 NaN"
    assert out.max() == pytest.approx(0.8)


# ---------------------------------------------------------------------------
# 口径统一：筛选 / 画像 / 回测 / Web 必须用同一份参数
# ---------------------------------------------------------------------------


def test_ttm_params_is_single_source_of_truth() -> None:
    """四处调用点必须都从 ttm_params() 取参，不得各自硬编码。"""
    import inspect
    from pathlib import Path

    root = Path(__file__).resolve().parents[1] / "src" / "hdiv"
    # 回测引擎曾硬编码 ttm_days=365, grace_days=45
    eng = (root / "backtest" / "engine.py").read_text(encoding="utf-8")
    assert "ttm_days=365, grace_days=45" not in eng, "引擎仍在硬编码 TTM 参数"
    assert "ttm_params()" in eng
    # walk-forward 与 web 曾用函数默认值
    for rel in ("backtest/walk_forward.py", "web/analysis.py"):
        text = (root / rel).read_text(encoding="utf-8")
        assert "ttm_params()" in text, f"{rel} 未使用统一参数"
    # 因子层自身
    f = inspect.getsource(__import__(
        "hdiv.factor.dividend_yield", fromlist=["x"]))
    assert "def ttm_params" in f


def test_ttm_dps_at_matches_series_right_endpoint() -> None:
    """单点求值（筛选器用）必须与序列右端点一致。"""
    d = _daily("2020-01-01", "2022-12-31")
    ev = _events([("2021-06-01", 1.0), ("2022-05-27", 1.2)])
    asof = pd.Timestamp("2021-12-31").date()
    one = ttm_dps_at(asof, ev)
    ser = ttm_dps_series(pd.DatetimeIndex([pd.Timestamp(asof)]), ev,
                         ttm_days=TTM, grace_days=GRACE, smooth_spikes=True)
    assert one is not None
    assert abs(one - float(ser[0])) < 1e-9


def test_ttm_params_reads_config() -> None:
    from hdiv.core.config import load_config

    w, g, sm = ttm_params()
    c = load_config("profile").ttm_dividend
    assert (w, g, sm) == (c.window_days, c.grace_days, c.smooth_spikes)


def test_config_exposes_smooth_spikes_switch() -> None:
    """开关必须暴露在 YAML 里，用户可自行关闭。"""
    from hdiv.core.config import load_config

    assert hasattr(load_config("profile").ttm_dividend, "smooth_spikes")


@pytest.mark.parametrize("grace", [0, 10, 45, 90])
def test_smoothing_never_produces_negative_or_nan(grace: int) -> None:
    d = _daily("2020-01-01", "2023-12-31")
    ev = _events([("2021-06-01", 1.0), ("2022-06-06", 1.2), ("2023-06-01", 1.4)])
    out = ttm_dps_series(d, ev, ttm_days=TTM, grace_days=grace, smooth_spikes=True)
    assert np.isfinite(out).all()
    assert (out >= 0).all()


# ---------------------------------------------------------------------------
# 透传一致性：包装函数必须接收并转发所有参数
# ---------------------------------------------------------------------------


def test_wrapper_signature_forwards_all_params() -> None:
    """回归：`dividend_yield_series` 是 `ttm_dps_series` 的包装。

    曾经只给**调用方**加了 `smooth_spikes`，却忘了在包装函数签名里声明，
    于是 walk-forward 直接 `TypeError` 崩在第一个窗口 —— 而测试全绿，
    因为测试没走 walk-forward 那条路径。
    """
    import inspect

    from hdiv.factor import dividend_yield as dy

    inner = set(inspect.signature(dy.ttm_dps_series).parameters) - {"dates", "events"}
    outer = set(inspect.signature(dy.dividend_yield_series).parameters) - {"close", "events"}
    missing = inner - outer
    assert not missing, (
        f"dividend_yield_series 未转发参数 {sorted(missing)}；"
        "调用方传了就会 TypeError"
    )
    # 且必须真的往下传
    src = inspect.getsource(dy.dividend_yield_series)
    for name in inner:
        assert f"{name}={name}" in src, f"包装函数未把 {name} 传给 ttm_dps_series"


def test_wrapper_accepts_smooth_spikes() -> None:
    """直接以关键字调用，确保签名真的可用（不只是字符串包含）。"""
    from hdiv.factor.dividend_yield import dividend_yield_series

    d = _daily("2021-01-01", "2022-12-31")
    close = pd.Series(10.0, index=d)
    ev = _events([("2021-06-01", 1.0), ("2022-05-27", 1.2)])
    for flag in (True, False):
        out = dividend_yield_series(close, ev, ttm_days=365, grace_days=45,
                                    smooth_spikes=flag)
        assert not out.empty
        assert "dividend_yield" in out.columns


def test_all_ttm_callers_pass_the_unified_params() -> None:
    """五处调用点都必须显式传 smooth_spikes，不能靠默认值（否则与配置脱钩）。"""
    from pathlib import Path

    root = Path(__file__).resolve().parents[1] / "src" / "hdiv"
    for rel in ("profile/builder.py", "backtest/engine.py",
                "backtest/walk_forward.py", "web/analysis.py"):
        src = (root / rel).read_text(encoding="utf-8")
        assert "smooth_spikes" in src, f"{rel} 未传 smooth_spikes（会与配置脱钩）"