初始提交：高股息策略研究与回测系统

从 Point-in-Time 股票筛选到统一 Web 前端的完整链路：筛选 → 画像 → 策略 → 回测 → Walk-forward → 绩效分析 → 报告/前端。架构 - 数据层与策略层分离；策略代码不写 SQL，只经 data/repo.py 取数 - 所有业务阈值集中在 config/*.yml，代码零硬编码（字段写错直接报错） - 报告只做「run_id → SQL → 渲染」，不做任何计算，数字可追溯 - 前后端分离：output/ 静态站点 + hdiv web 提供的 REST API 数据安全 - 只增不删：SQL 钩子拦截 DELETE/DROP/TRUNCATE，并有源码扫描测试守护 - qlib 原有表只读，本项目数据写入 hd_ 前缀表 - 回补使用 INSERT IGNORE，保证既有行零改动 - .env 存密钥且已 gitignore；output/、logs/、.venv/ 不入库交付物 - 30 张 hd_* 表、7 个 YAML 配置、283 项自动化测试 - 统一 Web 前端（hash 路由 SPA）+ nginx 部署配置与 launchd 托管脚本如实声明的限制 - 策略缺少稳定的样本外超额收益（Walk-forward 7 窗口均值 -0.95%，基准 +2.29%）；其价值体现在回撤控制，而非超额收益 - 涨跌停/停牌约束仅覆盖 2019 年起；index_weight 尚未填充 - AI Agent 层（plan.md 第四版 P8）未实现详见 docs/user-guide.md 与 docs/implementation-status.md。
2026-10-03 13:54:56 +08:00
commit fce725e13c
127 changed files with 28001 additions and 0 deletions
@@ -0,0 +1,264 @@
+"""同步层测试：值转换、去重键、数据帧构造、限频器。"""
+
+from __future__ import annotations
+
+from datetime import date, datetime
+
+import pandas as pd
+import pytest
+
+from hdiv.data.db import _frame_to_records
+from hdiv.data.sync.base import nullify_zero, stable_id, to_date, to_float
+from hdiv.data.tushare_client import RateLimiter
+
+
+# ---------------------------------------------------------------------------
+# 值转换
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [
+        ("20240102", date(2024, 1, 2)),
+        ("19991231", date(1999, 12, 31)),
+        (date(2020, 5, 6), date(2020, 5, 6)),
+        (datetime(2020, 5, 6, 1, 2, 3), date(2020, 5, 6)),
+        ("2020-05-06", date(2020, 5, 6)),
+        (None, None),
+        ("", None),
+        ("nan", None),
+        (float("nan"), None),
+    ],
+)
+def test_to_date(raw, expected) -> None:
+    assert to_date(raw) == expected
+
+
+@pytest.mark.parametrize(
+    "raw,expected",
+    [(1, 1.0), ("2.5", 2.5), (None, None), ("", None), ("abc", None), (float("nan"), None)],
+)
+def test_to_float(raw, expected) -> None:
+    assert to_float(raw) == expected
+
+
+def test_nullify_zero_treats_zero_as_missing() -> None:
+    """Tushare 用 0 表示「未实施」，需转 None 以免污染统计。"""
+    assert nullify_zero(0) is None
+    assert nullify_zero("0") is None
+    assert nullify_zero(1.5) == 1.5
+    assert nullify_zero(None) is None
+
+
+def test_stable_id_is_deterministic() -> None:
+    a = stable_id("dividend", "600036.SH", "2020-01-01")
+    b = stable_id("dividend", "600036.SH", "2020-01-01")
+    c = stable_id("dividend", "600036.SH", "2020-01-02")
+    assert a == b != c
+    assert len(a) == 32  # 默认 length=32
+
+
+# ---------------------------------------------------------------------------
+# NaN → SQL NULL（曾导致 "nan can not be used with MySQL" 的真实 bug）
+# ---------------------------------------------------------------------------
+
+
+def test_frame_to_records_converts_nan_to_none() -> None:
+    df = pd.DataFrame(
+        {
+            "a": [1.0, float("nan"), 3.0],
+            "b": ["x", None, "z"],
+            "c": [pd.Timestamp("2024-01-01"), pd.NaT, pd.Timestamp("2024-01-03")],
+            "d": pd.Series([1, 2, 3], dtype="Int64"),
+        }
+    )
+    recs = _frame_to_records(df)
+    assert recs[0]["a"] == 1.0
+    assert recs[1]["a"] is None, "NaN 必须转成 None，否则 pymysql 报错"
+    assert recs[1]["b"] is None
+    assert recs[1]["c"] is None, "NaT 必须转成 None"
+    # object dtype 保证 None 不被强制回 NaN
+    assert all(not (isinstance(v, float) and v != v) for r in recs for v in r.values())
+
+
+def test_frame_to_records_preserves_dates_and_strings() -> None:
+    df = pd.DataFrame({"d": [date(2024, 1, 2)], "s": ["银行"], "n": [1.5]})
+    rec = _frame_to_records(df)[0]
+    assert rec["d"] == date(2024, 1, 2)
+    assert rec["s"] == "银行"
+    assert rec["n"] == 1.5
+
+
+# ---------------------------------------------------------------------------
+# 分红去重键
+# ---------------------------------------------------------------------------
+
+
+def test_dividend_dedup_key_handles_null_ann_date() -> None:
+    """ann_date 为 NULL 时必须有稳定的显式键 —— MySQL 唯一约束不约束 NULL。"""
+    from hdiv.data.sync.dividend import build_dedup_key
+
+    k1 = build_dedup_key("600036.SH", "20260630", "股东提议", None)
+    k2 = build_dedup_key("600036.SH", "20260630", "股东提议", None)
+    assert k1 == k2, "同样的 NULL 输入必须产生同样的键，否则会重复入库"
+    assert "NONE" in k1
+    assert k1 != build_dedup_key("600036.SH", "20260630", "预案", None)
+    assert k1 != build_dedup_key("600036.SH", "20250630", "股东提议", None)
+
+
+def test_dividend_dedup_key_distinguishes_ann_date() -> None:
+    from hdiv.data.sync.dividend import build_dedup_key
+
+    a = build_dedup_key("600036.SH", "20251231", "实施", "20260328")
+    b = build_dedup_key("600036.SH", "20251231", "实施", "20260626")
+    assert a != b, "同一报告期的多次公告必须区分（预案 vs 实施）"
+
+
+def test_dividend_rows_to_frame_filters_incomplete() -> None:
+    from hdiv.data.sync.dividend import rows_to_frame
+
+    rows = [
+        {"ts_code": "600036.SH", "end_date": "20231231", "ann_date": "20240301",
+         "div_proc": "实施", "cash_div_tax": 2.0, "ex_date": "20240710"},
+        {"ts_code": "600036.SH", "end_date": None, "div_proc": "实施"},      # 缺报告期 → 丢
+        {"ts_code": "", "end_date": "20231231", "div_proc": "实施"},          # 缺代码 → 丢
+        {"ts_code": "600036.SH", "end_date": "20231231", "div_proc": ""},     # 缺状态 → 丢
+        {"ts_code": "600036.SH", "end_date": "20230630", "ann_date": None,
+         "div_proc": "股东提议", "cash_div_tax": None},                        # 保留（状态完整）
+    ]
+    df = rows_to_frame(rows)
+    assert len(df) == 2
+    assert set(df["div_proc"]) == {"实施", "股东提议"}
+    assert df["symbol"].eq("600036.SH").all()
+    # DataFrame 层缺失值表现为 NaN（pandas 的 float64 语义），
+    # 但**写入数据库前**必须变成 NULL —— 这才是真正的约束（见 _frame_to_records）
+    recs = _frame_to_records(df)
+    proposer = next(r for r in recs if r["div_proc"] == "股东提议")
+    assert proposer["cash_div_tax"] is None
+
+
+# ---------------------------------------------------------------------------
+# 财报：ann_date 为空必须丢弃（PIT 纪律）
+# ---------------------------------------------------------------------------
+
+
+def test_financial_rows_drop_missing_ann_date() -> None:
+    from hdiv.data.sync.financial import SPECS, rows_to_frame
+
+    spec = SPECS["fina_indicator"]
+    rows = [
+        {"ts_code": "600036.SH", "end_date": "20231231", "ann_date": "20240301", "roe": 15.0},
+        {"ts_code": "600036.SH", "end_date": "20230630", "ann_date": None, "roe": 8.0},
+        {"ts_code": "600036.SH", "end_date": None, "ann_date": "20240101", "roe": 1.0},
+    ]
+    df, dropped = rows_to_frame(rows, spec)
+    assert len(df) == 1, "缺公告日的记录无法用于 PIT，必须丢弃"
+    assert dropped == 2
+    assert df.iloc[0]["roe"] == 15.0
+
+
+def test_financial_report_type_preserved() -> None:
+    from hdiv.data.sync.financial import SPECS, rows_to_frame
+
+    spec = SPECS["income"]
+    rows = [
+        {"ts_code": "600036.SH", "end_date": "20231231", "ann_date": "20240301",
+         "report_type": "1", "total_revenue": 100.0},
+        {"ts_code": "600036.SH", "end_date": "20231231", "ann_date": "20240301",
+         "report_type": "2", "total_revenue": 30.0},
+        {"ts_code": "600036.SH", "end_date": "20230331", "ann_date": "20240401",
+         "report_type": None, "total_revenue": 5.0},
+    ]
+    df, _ = rows_to_frame(rows, spec)
+    assert len(df) == 3, "report_type 不同不得被折叠（否则合并报表与单季混为一谈）"
+    assert set(df["report_type"]) == {"1", "2", "1"}, "缺省应为 1（合并报表）"
+
+
+# ---------------------------------------------------------------------------
+# 行情帧构造
+# ---------------------------------------------------------------------------
+
+
+def test_price_frames_map_tushare_columns() -> None:
+    from hdiv.data.sync.price import adj_frame, basic_frame, daily_frame
+
+    d = daily_frame([{"ts_code": "000001.SZ", "trade_date": "20150105", "open": 10,
+                      "high": 11, "low": 9, "close": 10.5, "vol": 100, "amount": 1000}])
+    assert list(d.columns) == ["symbol", "trade_date", "open", "high", "low",
+                               "close", "volume", "amount", "source", "adjust"]
+    assert d.iloc[0]["symbol"] == "000001.SZ"
+    assert d.iloc[0]["volume"] == 100, "Tushare 的 vol 列映射为 volume"
+
+    a = adj_frame([{"ts_code": "000001.SZ", "trade_date": "20150105", "adj_factor": 1.2}])
+    assert a.iloc[0]["factor"] == 1.2
+
+    b = basic_frame([{"ts_code": "000001.SZ", "trade_date": "20150105", "pe": 8.0,
+                      "dv_ttm": 4.5, "total_mv": 1e11}])
+    assert b.iloc[0]["pe"] == 8.0
+    assert b.iloc[0]["dv_ttm"] == 4.5
+    assert b.iloc[0]["total_mv"] == 1e11
+
+
+def test_price_frames_drop_invalid_rows() -> None:
+    from hdiv.data.sync.price import daily_frame
+
+    df = daily_frame([
+        {"ts_code": "000001.SZ", "trade_date": "20150105", "close": 10},
+        {"ts_code": "", "trade_date": "20150105", "close": 10},
+        {"ts_code": "000002.SZ", "trade_date": None, "close": 10},
+    ])
+    assert len(df) == 1
+
+
+# ---------------------------------------------------------------------------
+# 限频器
+# ---------------------------------------------------------------------------
+
+
+def test_rate_limiter_enforces_budget() -> None:
+    rl = RateLimiter(per_minute=3)
+    for _ in range(3):
+        rl.acquire()
+    assert len(rl._hits) == 3
+    rl.reset()
+    assert len(rl._hits) == 0
+
+
+def test_rate_limiter_cooldown_clears_window() -> None:
+    rl = RateLimiter(per_minute=2)
+    rl.acquire()
+    rl.acquire()
+    rl.cooldown(0.01)
+    assert len(rl._hits) == 0, "冷却必须清空窗口，否则会持续撞限频"
+
+
+def test_per_api_limits_are_independent() -> None:
+    """限频按接口计算：dividend 与 daily 应有不同额度。"""
+    from hdiv.data.tushare_client import TushareClient
+
+    from hdiv.data import db
+
+    db.load_dotenv_once()
+    cfg = __import__("hdiv.core.config", fromlist=["load_config"]).load_config("datasource")
+    ts = cfg.tushare
+    assert ts.limit_for("dividend") == 180
+    assert ts.limit_for("daily") == 480
+    assert ts.limit_for("未知接口") == ts.rate_limit_default
+    # 构造客户端需要 token；此处仅验证配置层
+    assert ts.rate_limit_cooldown_sec >= 60, "冷却必须覆盖 Tushare 的 60 秒滑动窗口"
+
+
+# ---------------------------------------------------------------------------
+# index_weight 的月度切分
+# ---------------------------------------------------------------------------
+
+
+def test_month_starts_split() -> None:
+    from hdiv.data.sync.index import _month_starts
+
+    segs = _month_starts(date(2024, 1, 15), date(2024, 4, 10))
+    assert segs[0] == (date(2024, 1, 15), date(2024, 1, 31))
+    assert segs[1] == (date(2024, 2, 1), date(2024, 2, 29))  # 闰年
+    assert segs[-1] == (date(2024, 4, 1), date(2024, 4, 10))
+    assert len(segs) == 4