simon
fb6608193b
说明：本提交**不是本轮会话所做**，而是工作区里此前遗留的未提交改动。
为把历史分开，先单独提交它，再提交本轮会话的修改。
包含：
- Web 前端：web/index.html、web/app.js（统一 SPA，含回测/画像/Walk-forward 页面）
- 后端接口：web/server.py 路由、web/analysis.py（新增个股分析）
- 报告层：report/format.py（新增统一数字格式化 NumFmt）、
report/{backtest,profile,sensitivity,universe,walkforward}_report.py 接入 NumFmt、
report/renderer.py
- 股息率口径：factor/dividend_yield.py（毛刺消除 smooth_spikes）
- 筛选：universe/selector.py、universe/filters/dividend.py
- 绩效/敏感性：analysis/performance.py、analysis/sensitivity.py
- 部署：deploy/install-service.sh
- 测试：tests/test_format.py、tests/test_dividend_smoothing.py（新增）、
tests/test_web.py、tests/test_universe.py
提交时全量测试 403 项通过。
278 lines
10 KiB
Python
278 lines
10 KiB
Python
"""股票池报告装配器（plan.md §34「股票池」页面）。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import date, datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pandas as pd
|
||
|
||
from hdiv.core.config import config_hash, load_config
|
||
from hdiv.report.format import NumFmt
|
||
from hdiv.data import db
|
||
from hdiv.data.repo import Repo
|
||
from hdiv.report.renderer import Provenance, Renderer, query
|
||
|
||
# 滤网中文说明（呈现用，非业务逻辑）
|
||
_FILTER_DESC = {
|
||
"market": "交易所 / 板块 / 上市年限 / 市值 / 流动性 / 当日可交易",
|
||
"risk": "ST / 退市 / 停牌 / 净资产为负 / 资产负债率（金融豁免）",
|
||
"dividend": "股息率 / 连续分红年数 / 窗口内分红次数 / 支付率 / FCF 覆盖",
|
||
"quality": "年均 ROE / ROIC / 毛利率 / 净利率 / 经营现金流对利润（金融豁免）",
|
||
}
|
||
|
||
|
||
def build_universe_report(run_id: str, *, cfg: Any = None, top_rejected: int = 400) -> Path:
|
||
if cfg is None:
|
||
db.load_dotenv_once()
|
||
cfg = load_config("datasource")
|
||
r = Renderer()
|
||
prov = Provenance()
|
||
|
||
head = query(
|
||
"SELECT run_id, name, asof_date, mode, candidate_count, member_count, "
|
||
" config_json, config_hash, data_version, stats_json, status, created_at "
|
||
"FROM hd_universe_run WHERE run_id = :rid",
|
||
{"rid": run_id}, prov=prov, label="股票池运行头", cfg=cfg,
|
||
)
|
||
if head.empty:
|
||
raise ValueError(f"找不到 run_id={run_id}")
|
||
h = head.iloc[0]
|
||
|
||
members_df = query(
|
||
"SELECT symbol, name, industry, passed, fail_stage, fail_reason, values_json, filter_json "
|
||
"FROM hd_universe_member WHERE run_id = :rid",
|
||
{"rid": run_id}, prov=prov, label="股票池成员", cfg=cfg,
|
||
)
|
||
selected = members_df[members_df["passed"] == 1].copy()
|
||
rejected = members_df[members_df["passed"] == 0].copy()
|
||
|
||
# 逐股指标（从 values_json 展开）
|
||
def vals(s: str) -> dict:
|
||
import json
|
||
|
||
try:
|
||
return json.loads(s) if s else {}
|
||
except Exception:
|
||
return {}
|
||
|
||
sel_vals = selected["values_json"].map(vals) if not selected.empty else pd.Series(dtype=object)
|
||
if not selected.empty:
|
||
selected["dividend_yield"] = sel_vals.map(lambda d: d.get("dividend_yield"))
|
||
selected["continuity"] = sel_vals.map(
|
||
lambda d: d.get("dividend_continuity_years")
|
||
)
|
||
selected["roe"] = sel_vals.map(lambda d: d.get("roe"))
|
||
selected["debt_ratio"] = sel_vals.map(lambda d: d.get("debt_ratio"))
|
||
selected["payout_ratio"] = sel_vals.map(lambda d: d.get("payout_ratio"))
|
||
selected = selected.sort_values("dividend_yield", ascending=False, na_position="last")
|
||
|
||
# 入选股的市场指标
|
||
market_cols = {}
|
||
if not selected.empty:
|
||
asof = pd.to_datetime(h["asof_date"]).date()
|
||
repo = Repo(cfg)
|
||
panel = repo.market_panel(asof)
|
||
m = panel.set_index("symbol")
|
||
for c in ("pe_ttm", "pb", "total_mv"):
|
||
market_cols[c] = selected["symbol"].map(
|
||
lambda s: float(m.at[s, c]) if s in m.index and pd.notna(m.at[s, c]) else None
|
||
)
|
||
|
||
member_rows = []
|
||
for i, row in selected.iterrows():
|
||
member_rows.append(
|
||
{
|
||
"symbol": row["symbol"],
|
||
"name": row["name"],
|
||
"industry": row["industry"] or "—",
|
||
"dividend_yield": _pct(row.get("dividend_yield")),
|
||
"pe_ttm": _num(market_cols.get("pe_ttm", pd.Series()).get(i), 2),
|
||
"pb": _num(market_cols.get("pb", pd.Series()).get(i), 2),
|
||
"total_mv": _yi(market_cols.get("total_mv", pd.Series()).get(i)),
|
||
"continuity": _num(row.get("continuity"), 0),
|
||
"roe": _pct(row.get("roe")),
|
||
"debt_ratio": _pct(row.get("debt_ratio")),
|
||
"payout": _pct(row.get("payout_ratio")),
|
||
}
|
||
)
|
||
|
||
# 滤网漏斗
|
||
stats = {}
|
||
import json as _json
|
||
|
||
try:
|
||
stats = _json.loads(h["stats_json"]) if h["stats_json"] else {}
|
||
except Exception:
|
||
stats = {}
|
||
incoming = int(h["candidate_count"])
|
||
stages = []
|
||
for name in ("market", "risk", "dividend", "quality"):
|
||
removed = int(stats.get(name, 0))
|
||
stages.append(
|
||
{
|
||
"name": name,
|
||
"incoming": incoming,
|
||
"removed": removed,
|
||
"survived": incoming - removed,
|
||
"rate": f"{removed / incoming:.1%}" if incoming else "—",
|
||
"desc": _FILTER_DESC.get(name, ""),
|
||
}
|
||
)
|
||
incoming -= removed
|
||
funnel = [{"name": "候选", "value": int(h["candidate_count"])}] + [
|
||
{"name": f"通过 {s['name']}", "value": s["survived"]} for s in stages
|
||
]
|
||
|
||
# 行业分布
|
||
industry_rows: list[dict[str, Any]] = []
|
||
industry_chart: list[dict[str, Any]] = []
|
||
if not selected.empty:
|
||
cand_by_ind = rejected["industry"].value_counts().to_dict()
|
||
sel_by_ind = selected["industry"].fillna("未知").value_counts()
|
||
tot = int(sel_by_ind.sum()) or 1
|
||
for ind, n in sel_by_ind.items():
|
||
cand_n = int(cand_by_ind.get(ind, 0)) + int(n)
|
||
industry_rows.append(
|
||
{
|
||
"industry": ind,
|
||
"selected": int(n),
|
||
"share": f"{n / tot:.1%}",
|
||
"candidates": cand_n,
|
||
"hit_rate": f"{n / cand_n:.1%}" if cand_n else "—",
|
||
}
|
||
)
|
||
industry_chart.append({"name": ind, "value": int(n)})
|
||
|
||
# 股息率直方图（入选 vs 候选）
|
||
yield_hist = None
|
||
all_vals = members_df["values_json"].map(vals)
|
||
cand_yield = pd.to_numeric(
|
||
all_vals.map(lambda d: d.get("dividend_yield")), errors="coerce"
|
||
)
|
||
sel_yield = pd.to_numeric(sel_vals.map(lambda d: d.get("dividend_yield")), errors="coerce") \
|
||
if not selected.empty else pd.Series(dtype=float)
|
||
if cand_yield.notna().sum() > 0:
|
||
edges = [0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.08, 0.10, 1.0]
|
||
labels = ["<1%", "1-2%", "2-3%", "3-4%", "4-5%", "5-6%", "6-8%", "8-10%", ">10%"]
|
||
cand_counts = pd.cut(cand_yield, bins=edges, labels=labels, right=False).value_counts()
|
||
sel_counts = pd.cut(sel_yield, bins=edges, labels=labels, right=False).value_counts()
|
||
yield_hist = {
|
||
"buckets": labels,
|
||
"series": [
|
||
{"name": "候选", "data": [int(cand_counts.get(x, 0)) for x in labels]},
|
||
{"name": "入选", "data": [int(sel_counts.get(x, 0)) for x in labels]},
|
||
],
|
||
}
|
||
|
||
rej = rejected.head(top_rejected)
|
||
rejected_rows = [
|
||
{
|
||
"symbol": row["symbol"],
|
||
"name": row["name"],
|
||
"industry": row["industry"] or "—",
|
||
"fail_stage": row["fail_stage"] or "—",
|
||
"fail_reason": row["fail_reason"] or "—",
|
||
}
|
||
for _, row in rej.iterrows()
|
||
]
|
||
|
||
asof_d = pd.to_datetime(h["asof_date"]).date()
|
||
filename = r.name_from("universe", asof=asof_d.isoformat(), run_id=run_id)
|
||
n_member = int(h["member_count"])
|
||
n_cand = int(h["candidate_count"])
|
||
|
||
return r.render(
|
||
"reports/universe.html",
|
||
{
|
||
"candidate_count": f"{n_cand:,}",
|
||
"member_count": n_member,
|
||
"pass_rate": f"{n_member / n_cand:.2%}" if n_cand else "—",
|
||
"asof_date": str(asof_d),
|
||
"warnings": _warnings(h, n_member),
|
||
"stages": stages,
|
||
"funnel": funnel,
|
||
"members": member_rows,
|
||
"industry_rows": industry_rows,
|
||
"industry_chart": industry_chart,
|
||
"yield_hist": yield_hist,
|
||
"rejected": rejected_rows,
|
||
"config_json": _pretty_json(h["config_json"]),
|
||
"config_hash": h["config_hash"],
|
||
"data_version": h["data_version"],
|
||
"provenance_count": len(prov.entries),
|
||
"provenance_pretty": _pretty_prov(prov),
|
||
"subtitle": f"筛选配置 {h['name']} · 求值时点 {asof_d}（Point-in-Time）",
|
||
},
|
||
filename,
|
||
report_type="universe",
|
||
title=f"股票池报告 · {asof_d}",
|
||
source_run_ids=[run_id],
|
||
provenance=prov,
|
||
cfg=cfg,
|
||
)
|
||
|
||
|
||
def _warnings(h: Any, n_member: int) -> list[str]:
|
||
out: list[str] = []
|
||
if n_member == 0:
|
||
out.append("股票池为空：筛选条件过严，或相关数据（分红/财报）尚未同步完成")
|
||
elif n_member < 10:
|
||
out.append(f"股票池仅 {n_member} 只，偏少；请确认财务与分红数据是否已全量同步")
|
||
if h["status"] == "WARN":
|
||
out.append("运行状态为 WARN，详见下方漏斗与配置")
|
||
return out
|
||
|
||
|
||
def _num(v: Any, dec: int = 2) -> str:
|
||
if v is None or (isinstance(v, float) and v != v) or pd.isna(v):
|
||
return "—"
|
||
try:
|
||
return f"{float(v):,.{dec}f}"
|
||
except (TypeError, ValueError):
|
||
return str(v)
|
||
|
||
|
||
def _pct(v: Any, dec: int | None = None) -> str:
|
||
"""百分比。``dec`` 显式给出时按其格式化，否则用配置的精度。
|
||
|
||
早期实现默认 dec=2 且从不读配置，于是 decimals.ratio 改了也没反应。
|
||
"""
|
||
if v is None or (isinstance(v, float) and v != v) or pd.isna(v):
|
||
return "—"
|
||
try:
|
||
if dec is None:
|
||
return NumFmt.from_config().pct(v)
|
||
return f"{float(v) * 100:.{dec}f}%"
|
||
except (TypeError, ValueError):
|
||
return str(v)
|
||
|
||
|
||
def _yi(v: Any) -> str:
|
||
if v is None or (isinstance(v, float) and v != v) or pd.isna(v):
|
||
return "—"
|
||
return f"{float(v) / 1e8:,.1f}亿"
|
||
|
||
|
||
def _pretty_json(s: Any) -> str:
|
||
import json
|
||
|
||
try:
|
||
return json.dumps(json.loads(s), ensure_ascii=False, indent=2)
|
||
except Exception:
|
||
return str(s)
|
||
|
||
|
||
def _pretty_prov(prov: Provenance) -> str:
|
||
import json
|
||
|
||
lines = []
|
||
for i, e in enumerate(prov.entries, 1):
|
||
lines.append(f"[{i}] {e['label']}")
|
||
lines.append(f" {e['sql']}")
|
||
if e["params"]:
|
||
lines.append(f" 参数: {json.dumps(e['params'], ensure_ascii=False, default=str)}")
|
||
lines.append("")
|
||
return "\n".join(lines) or "（无查询记录）"
|