ggx/tools/diag_dividend_artifact.py

"""股息率（TTM 每股分红）毛刺诊断脚本。

**用途**：定位「数据毛刺导致回测异常成交」这一类问题。
它不改动任何数据，只读库、只打印。

用法::

    export PYTHONPATH=src
    .venv/bin/python tools/diag_dividend_artifact.py --symbol 600690.SH \
        --start 2026-07-01 --end 2026-08-05

诊断三件事：

1. **同一除权日的重复分红记录**：``hd_dividend`` 写入侧刻意保留
   预案/股东大会通过/实施 全量记录（决策 D6），但查询侧把它们当成
   **多笔独立分红**，于是 ``cash_div_tax`` 被重复累加。
2. **TTM 每股分红的时间线**：逐交易日打印去重前 / 去重后的取值，
   毛刺会表现为「无任何真实现金事件的一天突然跳变」。
3. **异常成交反查**：给定回测 run_id，列出每一笔卖出当日 TTM 值的
   去重前/去重后差异，标出「去重后不再触发卖出」的笔数。
"""

from __future__ import annotations

import argparse
import sys
from datetime import date, timedelta
from pathlib import Path

import pandas as pd

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

from hdiv.data import db  # noqa: E402
from hdiv.factor.dividend_yield import (  # noqa: E402
    build_dps_events,
    ttm_dps_series,
    ttm_params,
)


def _dedup(events: pd.DataFrame) -> pd.DataFrame:
    """同一除权日只保留一笔（取金额最大者）。

    这是「写入口径全量保留、查询口径按经济事件聚合」的最小实现。
    """
    if events.empty:
        return events
    return (
        events.sort_values("cash_div_tax")
        .drop_duplicates("ex_date", keep="last")
        .reset_index(drop=True)
    )


def diagnose_symbol(symbol: str, start: date, end: date) -> None:
    div = db.read_sql(
        "SELECT symbol, end_date, ann_date, imp_ann_date, div_proc, "
        "       cash_div_tax, ex_date "
        "FROM hd_dividend WHERE symbol = :s AND cash_div_tax > 0 "
        "  AND ex_date IS NOT NULL AND div_proc = '实施' "
        "ORDER BY ex_date",
        {"s": symbol},
    )
    events = build_dps_events(div).get(symbol)
    if events is None or events.empty:
        print(f"{symbol}: 无已实施现金分红记录")
        return

    dups = events.groupby("ex_date").size()
    dups = dups[dups > 1]
    print(f"=== {symbol} 分红记录 ===")
    print(f"已实施现金分红行数：{len(events)}，唯一除权日：{events['ex_date'].nunique()}")
    print(f"**同一除权日重复 {len(dups)} 组**（这些金额被重复累加）：")
    for ex, n in dups.items():
        rows = events[events["ex_date"] == ex]
        print(
            f"  {ex.date()}  x{n}  金额={rows['cash_div_tax'].tolist()}  "
            f"实施日={[str(pd.Timestamp(x).date()) for x in rows['imp_ann_date']]}"
        )

    px = db.read_sql(
        "SELECT trade_date, close FROM stock_daily "
        "WHERE symbol = :s AND trade_date BETWEEN :a AND :b ORDER BY trade_date",
        {"s": symbol, "a": start, "b": end},
    )
    if px.empty:
        print("区间内无行情")
        return
    px["trade_date"] = pd.to_datetime(px["trade_date"])
    idx = pd.DatetimeIndex(px["trade_date"])
    w, g, sm = ttm_params()
    raw = ttm_dps_series(idx, events, ttm_days=w, grace_days=g, smooth_spikes=sm)
    ded = ttm_dps_series(idx, _dedup(events), ttm_days=w, grace_days=g, smooth_spikes=sm)
    out = pd.DataFrame(
        {
            "trade_date": px["trade_date"].dt.date,
            "close": px["close"],
            "ttm_dps(去重前)": raw,
            "ttm_dps(去重后)": ded,
        }
    )
    out["股息率%(去重前)"] = (out["ttm_dps(去重前)"] / out["close"] * 100).round(3)
    out["股息率%(去重后)"] = (out["ttm_dps(去重后)"] / out["close"] * 100).round(3)
    print("\n=== 逐交易日 TTM 每股分红 ===")
    print(out.to_string(index=False))

    jump = out[pd.Series(raw, index=range(len(out))).diff().abs() > 1e-9]
    print("\n出现跳变的交易日（应与真实分红除权日一一对应）：")
    print(jump[["trade_date", "ttm_dps(去重前)", "ttm_dps(去重后)"]].to_string(index=False))


def diagnose_run(run_id: str) -> None:
    """反查某次回测的每一笔卖出：去重后是否仍会触发。"""
    from hdiv.data.repo import Repo

    repo = Repo()
    trades = db.read_sql(
        "SELECT symbol, signal_date, execution_date, reason_json "
        "FROM hd_backtest_trade WHERE run_id = :r AND side = 'SELL' "
        "ORDER BY execution_date",
        {"r": run_id},
    )
    if trades.empty:
        print("该 run 无卖出成交")
        return
    w, g, sm = ttm_params()
    years = 5
    rows = []
    for _, t in trades.iterrows():
        sym = t["symbol"]
        d0 = pd.Timestamp(t["signal_date"]).date()
        div = repo.dividend_records(d0, years_back=years + 3)
        div = div[div["symbol"] == sym]
        ev = build_dps_events(div).get(sym)
        if ev is None or ev.empty:
            continue
        px = repo.price_history([sym], date(d0.year - years - 1, 1, 1), d0, adjust="none")
        px = px.sort_values("trade_date")
        px["trade_date"] = pd.to_datetime(px["trade_date"])
        s = px.set_index("trade_date")["close"]
        i = pd.DatetimeIndex(s.index)
        ref = (d0 - timedelta(days=int(365.25 * years)), d0)

        def decision(events: pd.DataFrame) -> tuple[float, float, float]:
            dps = pd.Series(
                ttm_dps_series(i, events, ttm_days=w, grace_days=g, smooth_spikes=sm),
                index=i,
            )
            y = (dps / s).loc[: pd.Timestamp(d0)]
            cur = float(y.iloc[-1])
            rs = y.loc[pd.Timestamp(ref[0]) : pd.Timestamp(ref[1])]
            pct = float((rs <= cur).sum() / rs.size * 100) if rs.size else float("nan")
            return cur, pct, float(rs.quantile(0.25)) if rs.size else float("nan")

        for label, evx in (("去重前", ev), ("去重后", _dedup(ev))):
            cur, pct, p25 = decision(evx)
            rows.append(
                {
                    "symbol": sym,
                    "signal_date": d0,
                    "口径": label,
                    "股息率%": round(cur * 100, 3),
                    "分位%": round(pct, 2),
                    "P25%": round(p25 * 100, 3),
                    "触发卖出(P25)": pct <= 25.0,
                }
            )
    if not rows:
        return
    df = pd.DataFrame(rows)
    print(f"\n=== run {run_id} 卖出决策：去重前 vs 去重后 ===")
    print(df.pivot_table(
        index=["symbol", "signal_date"], columns="口径",
        values=["分位%", "触发卖出(P25)"], aggfunc="first",
    ).to_string())
    before = df[df["口径"] == "去重前"].set_index(["symbol", "signal_date"])["触发卖出(P25)"]
    after = df[df["口径"] == "去重后"].set_index(["symbol", "signal_date"])["触发卖出(P25)"]
    flipped = before[before & ~after.reindex(before.index).fillna(False)]
    print(f"\n去重后**不再触发**卖出的成交：{len(flipped)} / {len(before)} 笔")
    for k in flipped.index:
        print(f"  {k[0]}  信号日 {k[1]}")


def main() -> int:
    ap = argparse.ArgumentParser(description="股息率毛刺诊断")
    ap.add_argument("--symbol", help="股票代码，如 600690.SH")
    ap.add_argument("--start", help="起始日 YYYY-MM-DD")
    ap.add_argument("--end", help="结束日 YYYY-MM-DD")
    ap.add_argument("--run-id", help="反查某次回测的所有卖出成交")
    ap.add_argument("--duplicate-survey", action="store_true",
                    help="全库统计：同一除权日重复记录的股票数")
    args = ap.parse_args()

    db.load_dotenv_once()

    if args.duplicate_survey:
        df = db.read_sql(
            "SELECT symbol, COUNT(*) AS rows_, COUNT(DISTINCT ex_date) AS ex_dates "
            "FROM hd_dividend WHERE cash_div_tax > 0 AND ex_date IS NOT NULL "
            "  AND div_proc = '实施' GROUP BY symbol",
            {},
        )
        bad = df[df["rows_"] > df["ex_dates"]]
        print(f"有已实施现金分红的股票：{len(df)}")
        print(f"**存在同一除权日重复记录的股票：{len(bad)}**"
              f"（多出 {int((bad['rows_'] - bad['ex_dates']).sum())} 行被重复累加）")
        print(bad.sort_values("rows_", ascending=False).head(20).to_string(index=False))
        return 0

    if args.run_id:
        diagnose_run(args.run_id)
    if args.symbol:
        start = date.fromisoformat(args.start) if args.start else date(2026, 1, 1)
        end = date.fromisoformat(args.end) if args.end else date(2026, 12, 31)
        diagnose_symbol(args.symbol, start, end)
    if not (args.run_id or args.symbol or args.duplicate_survey):
        ap.print_help()
    return 0


if __name__ == "__main__":
    raise SystemExit(main())