初始提交：高股息策略研究与回测系统
从 Point-in-Time 股票筛选到统一 Web 前端的完整链路： 筛选 → 画像 → 策略 → 回测 → Walk-forward → 绩效分析 → 报告/前端。 架构 - 数据层与策略层分离；策略代码不写 SQL，只经 data/repo.py 取数 - 所有业务阈值集中在 config/*.yml，代码零硬编码（字段写错直接报错） - 报告只做「run_id → SQL → 渲染」，不做任何计算，数字可追溯 - 前后端分离：output/ 静态站点 + hdiv web 提供的 REST API 数据安全 - 只增不删：SQL 钩子拦截 DELETE/DROP/TRUNCATE，并有源码扫描测试守护 - qlib 原有表只读，本项目数据写入 hd_ 前缀表 - 回补使用 INSERT IGNORE，保证既有行零改动 - .env 存密钥且已 gitignore；output/、logs/、.venv/ 不入库 交付物 - 30 张 hd_* 表、7 个 YAML 配置、283 项自动化测试 - 统一 Web 前端（hash 路由 SPA）+ nginx 部署配置与 launchd 托管脚本 如实声明的限制 - 策略缺少稳定的样本外超额收益（Walk-forward 7 窗口均值 -0.95%， 基准 +2.29%）；其价值体现在回撤控制，而非超额收益 - 涨跌停/停牌约束仅覆盖 2019 年起；index_weight 尚未填充 - AI Agent 层（plan.md 第四版 P8）未实现 详见 docs/user-guide.md 与 docs/implementation-status.md。
This commit is contained in:
@@ -0,0 +1,270 @@
|
||||
"""股票池报告装配器（plan.md §34「股票池」页面）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from hdiv.core.config import config_hash, load_config
|
||||
from hdiv.data import db
|
||||
from hdiv.data.repo import Repo
|
||||
from hdiv.report.renderer import Provenance, Renderer, query
|
||||
|
||||
# 滤网中文说明（呈现用，非业务逻辑）
|
||||
_FILTER_DESC = {
|
||||
"market": "交易所 / 板块 / 上市年限 / 市值 / 流动性 / 当日可交易",
|
||||
"risk": "ST / 退市 / 停牌 / 净资产为负 / 资产负债率（金融豁免）",
|
||||
"dividend": "股息率 / 连续分红年数 / 窗口内分红次数 / 支付率 / FCF 覆盖",
|
||||
"quality": "年均 ROE / ROIC / 毛利率 / 净利率 / 经营现金流对利润（金融豁免）",
|
||||
}
|
||||
|
||||
|
||||
def build_universe_report(run_id: str, *, cfg: Any = None, top_rejected: int = 400) -> Path:
|
||||
if cfg is None:
|
||||
db.load_dotenv_once()
|
||||
cfg = load_config("datasource")
|
||||
r = Renderer()
|
||||
prov = Provenance()
|
||||
|
||||
head = query(
|
||||
"SELECT run_id, name, asof_date, mode, candidate_count, member_count, "
|
||||
" config_json, config_hash, data_version, stats_json, status, created_at "
|
||||
"FROM hd_universe_run WHERE run_id = :rid",
|
||||
{"rid": run_id}, prov=prov, label="股票池运行头", cfg=cfg,
|
||||
)
|
||||
if head.empty:
|
||||
raise ValueError(f"找不到 run_id={run_id}")
|
||||
h = head.iloc[0]
|
||||
|
||||
members_df = query(
|
||||
"SELECT symbol, name, industry, passed, fail_stage, fail_reason, values_json, filter_json "
|
||||
"FROM hd_universe_member WHERE run_id = :rid",
|
||||
{"rid": run_id}, prov=prov, label="股票池成员", cfg=cfg,
|
||||
)
|
||||
selected = members_df[members_df["passed"] == 1].copy()
|
||||
rejected = members_df[members_df["passed"] == 0].copy()
|
||||
|
||||
# 逐股指标（从 values_json 展开）
|
||||
def vals(s: str) -> dict:
|
||||
import json
|
||||
|
||||
try:
|
||||
return json.loads(s) if s else {}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
sel_vals = selected["values_json"].map(vals) if not selected.empty else pd.Series(dtype=object)
|
||||
if not selected.empty:
|
||||
selected["dividend_yield"] = sel_vals.map(lambda d: d.get("dividend_yield"))
|
||||
selected["continuity"] = sel_vals.map(
|
||||
lambda d: d.get("dividend_continuity_years")
|
||||
)
|
||||
selected["roe"] = sel_vals.map(lambda d: d.get("roe"))
|
||||
selected["debt_ratio"] = sel_vals.map(lambda d: d.get("debt_ratio"))
|
||||
selected["payout_ratio"] = sel_vals.map(lambda d: d.get("payout_ratio"))
|
||||
selected = selected.sort_values("dividend_yield", ascending=False, na_position="last")
|
||||
|
||||
# 入选股的市场指标
|
||||
market_cols = {}
|
||||
if not selected.empty:
|
||||
asof = pd.to_datetime(h["asof_date"]).date()
|
||||
repo = Repo(cfg)
|
||||
panel = repo.market_panel(asof)
|
||||
m = panel.set_index("symbol")
|
||||
for c in ("pe_ttm", "pb", "total_mv"):
|
||||
market_cols[c] = selected["symbol"].map(
|
||||
lambda s: float(m.at[s, c]) if s in m.index and pd.notna(m.at[s, c]) else None
|
||||
)
|
||||
|
||||
member_rows = []
|
||||
for i, row in selected.iterrows():
|
||||
member_rows.append(
|
||||
{
|
||||
"symbol": row["symbol"],
|
||||
"name": row["name"],
|
||||
"industry": row["industry"] or "—",
|
||||
"dividend_yield": _pct(row.get("dividend_yield")),
|
||||
"pe_ttm": _num(market_cols.get("pe_ttm", pd.Series()).get(i), 2),
|
||||
"pb": _num(market_cols.get("pb", pd.Series()).get(i), 2),
|
||||
"total_mv": _yi(market_cols.get("total_mv", pd.Series()).get(i)),
|
||||
"continuity": _num(row.get("continuity"), 0),
|
||||
"roe": _pct(row.get("roe")),
|
||||
"debt_ratio": _pct(row.get("debt_ratio")),
|
||||
"payout": _pct(row.get("payout_ratio")),
|
||||
}
|
||||
)
|
||||
|
||||
# 滤网漏斗
|
||||
stats = {}
|
||||
import json as _json
|
||||
|
||||
try:
|
||||
stats = _json.loads(h["stats_json"]) if h["stats_json"] else {}
|
||||
except Exception:
|
||||
stats = {}
|
||||
incoming = int(h["candidate_count"])
|
||||
stages = []
|
||||
for name in ("market", "risk", "dividend", "quality"):
|
||||
removed = int(stats.get(name, 0))
|
||||
stages.append(
|
||||
{
|
||||
"name": name,
|
||||
"incoming": incoming,
|
||||
"removed": removed,
|
||||
"survived": incoming - removed,
|
||||
"rate": f"{removed / incoming:.1%}" if incoming else "—",
|
||||
"desc": _FILTER_DESC.get(name, ""),
|
||||
}
|
||||
)
|
||||
incoming -= removed
|
||||
funnel = [{"name": "候选", "value": int(h["candidate_count"])}] + [
|
||||
{"name": f"通过 {s['name']}", "value": s["survived"]} for s in stages
|
||||
]
|
||||
|
||||
# 行业分布
|
||||
industry_rows: list[dict[str, Any]] = []
|
||||
industry_chart: list[dict[str, Any]] = []
|
||||
if not selected.empty:
|
||||
cand_by_ind = rejected["industry"].value_counts().to_dict()
|
||||
sel_by_ind = selected["industry"].fillna("未知").value_counts()
|
||||
tot = int(sel_by_ind.sum()) or 1
|
||||
for ind, n in sel_by_ind.items():
|
||||
cand_n = int(cand_by_ind.get(ind, 0)) + int(n)
|
||||
industry_rows.append(
|
||||
{
|
||||
"industry": ind,
|
||||
"selected": int(n),
|
||||
"share": f"{n / tot:.1%}",
|
||||
"candidates": cand_n,
|
||||
"hit_rate": f"{n / cand_n:.1%}" if cand_n else "—",
|
||||
}
|
||||
)
|
||||
industry_chart.append({"name": ind, "value": int(n)})
|
||||
|
||||
# 股息率直方图（入选 vs 候选）
|
||||
yield_hist = None
|
||||
all_vals = members_df["values_json"].map(vals)
|
||||
cand_yield = pd.to_numeric(
|
||||
all_vals.map(lambda d: d.get("dividend_yield")), errors="coerce"
|
||||
)
|
||||
sel_yield = pd.to_numeric(sel_vals.map(lambda d: d.get("dividend_yield")), errors="coerce") \
|
||||
if not selected.empty else pd.Series(dtype=float)
|
||||
if cand_yield.notna().sum() > 0:
|
||||
edges = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.08, 0.10, 1.0]
|
||||
labels = ["<1%", "1-2%", "2-3%", "3-4%", "4-5%", "5-6%", "6-8%", "8-10%", ">10%"]
|
||||
cand_counts = pd.cut(cand_yield, bins=edges, labels=labels, right=False).value_counts()
|
||||
sel_counts = pd.cut(sel_yield, bins=edges, labels=labels, right=False).value_counts()
|
||||
yield_hist = {
|
||||
"buckets": labels,
|
||||
"series": [
|
||||
{"name": "候选", "data": [int(cand_counts.get(x, 0)) for x in labels]},
|
||||
{"name": "入选", "data": [int(sel_counts.get(x, 0)) for x in labels]},
|
||||
],
|
||||
}
|
||||
|
||||
rej = rejected.head(top_rejected)
|
||||
rejected_rows = [
|
||||
{
|
||||
"symbol": row["symbol"],
|
||||
"name": row["name"],
|
||||
"industry": row["industry"] or "—",
|
||||
"fail_stage": row["fail_stage"] or "—",
|
||||
"fail_reason": row["fail_reason"] or "—",
|
||||
}
|
||||
for _, row in rej.iterrows()
|
||||
]
|
||||
|
||||
asof_d = pd.to_datetime(h["asof_date"]).date()
|
||||
filename = r.name_from("universe", asof=asof_d.isoformat(), run_id=run_id)
|
||||
n_member = int(h["member_count"])
|
||||
n_cand = int(h["candidate_count"])
|
||||
|
||||
return r.render(
|
||||
"reports/universe.html",
|
||||
{
|
||||
"candidate_count": f"{n_cand:,}",
|
||||
"member_count": n_member,
|
||||
"pass_rate": f"{n_member / n_cand:.2%}" if n_cand else "—",
|
||||
"asof_date": str(asof_d),
|
||||
"warnings": _warnings(h, n_member),
|
||||
"stages": stages,
|
||||
"funnel": funnel,
|
||||
"members": member_rows,
|
||||
"industry_rows": industry_rows,
|
||||
"industry_chart": industry_chart,
|
||||
"yield_hist": yield_hist,
|
||||
"rejected": rejected_rows,
|
||||
"config_json": _pretty_json(h["config_json"]),
|
||||
"config_hash": h["config_hash"],
|
||||
"data_version": h["data_version"],
|
||||
"provenance_count": len(prov.entries),
|
||||
"provenance_pretty": _pretty_prov(prov),
|
||||
"subtitle": f"筛选配置 {h['name']} · 求值时点 {asof_d}（Point-in-Time）",
|
||||
},
|
||||
filename,
|
||||
report_type="universe",
|
||||
title=f"股票池报告 · {asof_d}",
|
||||
source_run_ids=[run_id],
|
||||
provenance=prov,
|
||||
cfg=cfg,
|
||||
)
|
||||
|
||||
|
||||
def _warnings(h: Any, n_member: int) -> list[str]:
|
||||
out: list[str] = []
|
||||
if n_member == 0:
|
||||
out.append("股票池为空：筛选条件过严，或相关数据（分红/财报）尚未同步完成")
|
||||
elif n_member < 10:
|
||||
out.append(f"股票池仅 {n_member} 只，偏少；请确认财务与分红数据是否已全量同步")
|
||||
if h["status"] == "WARN":
|
||||
out.append("运行状态为 WARN，详见下方漏斗与配置")
|
||||
return out
|
||||
|
||||
|
||||
def _num(v: Any, dec: int = 2) -> str:
|
||||
if v is None or (isinstance(v, float) and v != v) or pd.isna(v):
|
||||
return "—"
|
||||
try:
|
||||
return f"{float(v):,.{dec}f}"
|
||||
except (TypeError, ValueError):
|
||||
return str(v)
|
||||
|
||||
|
||||
def _pct(v: Any, dec: int = 2) -> str:
|
||||
if v is None or (isinstance(v, float) and v != v) or pd.isna(v):
|
||||
return "—"
|
||||
try:
|
||||
return f"{float(v) * 100:.{dec}f}%"
|
||||
except (TypeError, ValueError):
|
||||
return str(v)
|
||||
|
||||
|
||||
def _yi(v: Any) -> str:
|
||||
if v is None or (isinstance(v, float) and v != v) or pd.isna(v):
|
||||
return "—"
|
||||
return f"{float(v) / 1e8:,.1f}亿"
|
||||
|
||||
|
||||
def _pretty_json(s: Any) -> str:
|
||||
import json
|
||||
|
||||
try:
|
||||
return json.dumps(json.loads(s), ensure_ascii=False, indent=2)
|
||||
except Exception:
|
||||
return str(s)
|
||||
|
||||
|
||||
def _pretty_prov(prov: Provenance) -> str:
|
||||
import json
|
||||
|
||||
lines = []
|
||||
for i, e in enumerate(prov.entries, 1):
|
||||
lines.append(f"[{i}] {e['label']}")
|
||||
lines.append(f" {e['sql']}")
|
||||
if e["params"]:
|
||||
lines.append(f" 参数: {json.dumps(e['params'], ensure_ascii=False, default=str)}")
|
||||
lines.append("")
|
||||
return "\n".join(lines) or "（无查询记录）"
|
||||
Reference in New Issue
Block a user