Initial commit: cc-cursor 全链路量化研究平台
7 Sprints 全部完成: Sprint 0: 基础设施 (DataManager + MariaDB) Sprint 1: 因子引擎 (34因子/12分类) Sprint 2: VectorBT 回测 (5策略+截面) Sprint 3: Optuna 优化 (+Walk-Forward) Sprint 4: ML 模型 (LightGBM+CatBoost) Sprint 5: Qwen 情绪因子 (三源新闻+日期对齐) Sprint 6: Agent 系统 (4Agent+日报.md/.html) 生产加固 (15项): Tushare双源fallback, SSH自动恢复, pool_pre_ping, save_daily先删后插, load_dotenv绝对路径, 日报5d/20d修复, RiskAgent改上证指数, 昨日对比+数据截止, mac_report utf8mb4, CLAUDE-*.md 9条已知Bug, demo全参数化, djapi数据源归一化, indexDatas API修正 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
ML 模型回测集成。
|
||||
|
||||
MLStrategy: 将 ML 预测值作为交易信号接入回测引擎。
|
||||
MLBenchmark: 多模型基准对比。
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from backtest.base import BaseStrategy
|
||||
from backtest.vectorbt.engine import VectorBTEngine
|
||||
from models.base import BaseModel
|
||||
from models.features import FeatureEngine
|
||||
|
||||
|
||||
class MLStrategy(BaseStrategy):
|
||||
"""
|
||||
ML 预测 → 交易信号。
|
||||
|
||||
用模型预测未来 N 日收益，按预测值分位数生成信号：
|
||||
- 预测值 > buy_quantile → 买入
|
||||
- 预测值 < sell_quantile → 平仓
|
||||
|
||||
参数:
|
||||
model: 已训练的 BaseModel
|
||||
feature_engine: 已 fit 的 FeatureEngine
|
||||
buy_quantile: 买入分位阈值（0.7 = 预测值最高的30%买入）
|
||||
sell_quantile: 卖出分位阈值（0.3 = 预测值最低的30%平仓）
|
||||
rebalance_freq: 调仓间隔（交易日）
|
||||
"""
|
||||
|
||||
category = "ml"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: BaseModel,
|
||||
feature_engine: FeatureEngine,
|
||||
buy_quantile: float = 0.7,
|
||||
sell_quantile: float = 0.3,
|
||||
rebalance_freq: int = 5,
|
||||
):
|
||||
self.model = model
|
||||
self.feature_engine = feature_engine
|
||||
self.buy_quantile = buy_quantile
|
||||
self.sell_quantile = sell_quantile
|
||||
self.rebalance_freq = rebalance_freq
|
||||
self.name = f"ml_{model.name}"
|
||||
|
||||
def generate_signals(self, factor_df: pd.DataFrame) -> pd.Series:
|
||||
X, _ = self.feature_engine.build(factor_df, factor_df, fit=False)
|
||||
if X.empty:
|
||||
return pd.Series(-1, index=factor_df.index)
|
||||
|
||||
preds = self.model.predict(X)
|
||||
# 用预测值本身的分布作为阈值（相对排序，避免模型偏差影响）
|
||||
buy_threshold = preds.quantile(self.buy_quantile)
|
||||
sell_threshold = preds.quantile(self.sell_quantile)
|
||||
|
||||
signals = pd.Series(-1, index=factor_df.index)
|
||||
common = signals.index.intersection(preds.index)
|
||||
buy_mask = preds.loc[common] > buy_threshold
|
||||
sell_mask = preds.loc[common] < sell_threshold
|
||||
signals.loc[buy_mask[buy_mask].index] = 1
|
||||
signals.loc[sell_mask[sell_mask].index] = 0
|
||||
|
||||
signals = self._filter_rebalance(signals)
|
||||
return signals
|
||||
|
||||
def _filter_rebalance(self, signals: pd.Series) -> pd.Series:
|
||||
"""每隔 rebalance_freq 个交易日保留第一个非持有信号。"""
|
||||
result = signals.copy()
|
||||
last_active = -self.rebalance_freq - 1
|
||||
for i in range(len(result)):
|
||||
sig = result.iloc[i]
|
||||
if sig in (0, 1):
|
||||
if i - last_active >= self.rebalance_freq:
|
||||
last_active = i
|
||||
else:
|
||||
result.iloc[i] = -1
|
||||
return result
|
||||
|
||||
|
||||
class MLBenchmark:
|
||||
"""ML 模型基准对比测试。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
models: list[BaseModel],
|
||||
feature_engine: FeatureEngine,
|
||||
price_df: pd.DataFrame,
|
||||
factor_df: pd.DataFrame,
|
||||
bt_engine: VectorBTEngine | None = None,
|
||||
):
|
||||
self.models = models
|
||||
self.feature_engine = feature_engine
|
||||
self.price_df = price_df
|
||||
self.factor_df = factor_df
|
||||
self.bt_engine = bt_engine or VectorBTEngine()
|
||||
|
||||
def run(self) -> pd.DataFrame:
|
||||
"""对比各模型的预测质量和回测表现。"""
|
||||
rows = []
|
||||
for model in self.models:
|
||||
strategy = MLStrategy(model, self.feature_engine)
|
||||
report = self.bt_engine.run(strategy, self.price_df, self.factor_df)
|
||||
|
||||
# OOS 预测 vs 真实值
|
||||
X, y_true = self.feature_engine.build(self.factor_df, self.price_df, fit=True)
|
||||
y_pred = model.predict(X)
|
||||
|
||||
ic = y_pred.corr(y_true) if len(y_pred) > 0 else 0
|
||||
|
||||
rows.append({
|
||||
"model": model.name,
|
||||
"ic": round(ic, 4),
|
||||
"total_return": report.total_return,
|
||||
"cagr": report.cagr,
|
||||
"max_dd": report.max_drawdown,
|
||||
"sharpe": report.sharpe_ratio,
|
||||
"win_rate": report.win_rate,
|
||||
"trades": report.total_trades,
|
||||
})
|
||||
return pd.DataFrame(rows).set_index("model")
|
||||
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
ML 模型抽象基类。
|
||||
|
||||
统一接口：fit(X, y) → predict(X) → save/load。
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
import pickle
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
class BaseModel(ABC):
|
||||
"""ML 模型抽象基类。"""
|
||||
|
||||
name: str = ""
|
||||
|
||||
@abstractmethod
|
||||
def fit(self, X: pd.DataFrame, y: pd.Series) -> "BaseModel":
|
||||
"""训练模型。返回 self 支持链式调用。"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def predict(self, X: pd.DataFrame) -> pd.Series:
|
||||
"""返回预测值（回归值）。"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def get_feature_importance(self) -> pd.DataFrame:
|
||||
"""特征重要性 DataFrame，columns=[feature, importance]。"""
|
||||
...
|
||||
|
||||
def save(self, path: str) -> None:
|
||||
"""保存模型到文件（pickle）。"""
|
||||
with open(path, "wb") as f:
|
||||
pickle.dump(self, f)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: str) -> "BaseModel":
|
||||
"""从文件加载模型。"""
|
||||
with open(path, "rb") as f:
|
||||
return pickle.load(f)
|
||||
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
CatBoost 模型封装。
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from catboost import CatBoostRegressor, Pool
|
||||
|
||||
from models.base import BaseModel
|
||||
from sklearn.model_selection import TimeSeriesSplit
|
||||
|
||||
_DEFAULT_PARAMS = {
|
||||
"loss_function": "RMSE",
|
||||
"iterations": 1000,
|
||||
"learning_rate": 0.03,
|
||||
"depth": 5,
|
||||
"random_seed": 42,
|
||||
"verbose": False,
|
||||
"allow_writing_files": False,
|
||||
"min_data_in_leaf": 20,
|
||||
}
|
||||
|
||||
|
||||
class CatBoostModel(BaseModel):
|
||||
"""CatBoost 回归模型。"""
|
||||
|
||||
name = "catboost"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
params: dict | None = None,
|
||||
early_stopping: int = 50,
|
||||
eval_ratio: float = 0.2,
|
||||
random_seed: int = 42,
|
||||
):
|
||||
self.params = params or _DEFAULT_PARAMS.copy()
|
||||
self.params["random_seed"] = random_seed
|
||||
self.early_stopping = early_stopping
|
||||
self.eval_ratio = eval_ratio
|
||||
self._model: CatBoostRegressor | None = None
|
||||
self._feature_names: list[str] = []
|
||||
|
||||
def fit(self, X: pd.DataFrame, y: pd.Series) -> "CatBoostModel":
|
||||
self._feature_names = list(X.columns)
|
||||
|
||||
n = len(X)
|
||||
val_size = int(n * self.eval_ratio)
|
||||
|
||||
X_train, y_train = X, y
|
||||
eval_set = None
|
||||
|
||||
if val_size >= 50:
|
||||
split_idx = n - val_size
|
||||
X_train, X_val = X.iloc[:split_idx], X.iloc[split_idx:]
|
||||
y_train, y_val = y.iloc[:split_idx], y.iloc[split_idx:]
|
||||
eval_set = Pool(X_val, y_val)
|
||||
|
||||
self._model = CatBoostRegressor(**self.params)
|
||||
self._model.fit(
|
||||
X_train, y_train,
|
||||
eval_set=eval_set,
|
||||
early_stopping_rounds=self.early_stopping if eval_set else None,
|
||||
verbose=False,
|
||||
)
|
||||
return self
|
||||
|
||||
def predict(self, X: pd.DataFrame) -> pd.Series:
|
||||
if self._model is None:
|
||||
raise RuntimeError("模型尚未训练")
|
||||
preds = self._model.predict(X[self._feature_names])
|
||||
return pd.Series(preds, index=X.index, name="pred")
|
||||
|
||||
def get_feature_importance(self) -> pd.DataFrame:
|
||||
if self._model is None:
|
||||
return pd.DataFrame()
|
||||
imp = self._model.get_feature_importance()
|
||||
names = self._feature_names
|
||||
df = pd.DataFrame({"feature": names, "importance": imp})
|
||||
total = df["importance"].sum()
|
||||
df["importance_pct"] = df["importance"] / total * 100 if total > 0 else 0
|
||||
return df.sort_values("importance", ascending=False)
|
||||
|
||||
def cv_evaluate(
|
||||
self, X: pd.DataFrame, y: pd.Series, n_folds: int = 5
|
||||
) -> pd.DataFrame:
|
||||
"""时间序列交叉验证评估。"""
|
||||
tscv = TimeSeriesSplit(n_splits=n_folds)
|
||||
results = []
|
||||
for fold, (train_idx, test_idx) in enumerate(tscv.split(X)):
|
||||
X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
|
||||
y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]
|
||||
|
||||
model = CatBoostModel(
|
||||
params=self.params,
|
||||
early_stopping=self.early_stopping,
|
||||
eval_ratio=0.0,
|
||||
)
|
||||
model.fit(X_train, y_train)
|
||||
preds = model.predict(X_test)
|
||||
ic = preds.corr(y_test)
|
||||
mse = ((preds - y_test) ** 2).mean()
|
||||
results.append({"fold": fold, "ic": round(ic, 4), "mse": round(mse, 4)})
|
||||
|
||||
df = pd.DataFrame(results)
|
||||
df.loc["mean"] = df.mean()
|
||||
return df
|
||||
|
||||
@property
|
||||
def n_estimators_used(self) -> int | None:
|
||||
"""实际使用的树数量。"""
|
||||
if self._model is None:
|
||||
return None
|
||||
return self._model.tree_count_
|
||||
@@ -0,0 +1,142 @@
|
||||
"""
|
||||
特征工程：因子 → 特征矩阵 + 目标标签。
|
||||
|
||||
严禁使用未来数据。所有变换基于 expanding window 或训练集统计。
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.preprocessing import RobustScaler
|
||||
|
||||
|
||||
class FeatureEngine:
|
||||
"""
|
||||
特征工程引擎。
|
||||
|
||||
参数:
|
||||
lookahead: 预测未来 N 个交易日
|
||||
label_type: 'regression' | 'classification'
|
||||
winsorize_pct: 去极值的分位数边界 (0.01, 0.99)
|
||||
nan_threshold: NaN 占比超过此值的因子直接剔除
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
lookahead: int = 5,
|
||||
label_type: str = "regression",
|
||||
winsorize_pct: tuple[float, float] = (0.01, 0.99),
|
||||
nan_threshold: float = 0.3,
|
||||
):
|
||||
self.lookahead = lookahead
|
||||
self.label_type = label_type
|
||||
self.winsorize_pct = winsorize_pct
|
||||
self.nan_threshold = nan_threshold
|
||||
self._scaler = RobustScaler()
|
||||
self._scaler_fitted = False
|
||||
self._valid_features: list[str] = []
|
||||
|
||||
# ── 标签构建 ──────────────────────────────────────────
|
||||
|
||||
def build_labels(self, price_df: pd.DataFrame) -> pd.Series:
|
||||
"""
|
||||
构建目标标签。
|
||||
|
||||
regression: (close_{t+N} - close_t) / close_t * 100
|
||||
classification: 1 if return > 0 else 0
|
||||
"""
|
||||
close = price_df["close"]
|
||||
future = close.shift(-self.lookahead)
|
||||
ret = (future - close) / close * 100
|
||||
|
||||
if self.label_type == "classification":
|
||||
return (ret > 0).astype(int)
|
||||
|
||||
return ret.rename(f"y_fwd_{self.lookahead}")
|
||||
|
||||
# ── 特征构建 ──────────────────────────────────────────
|
||||
|
||||
def build(
|
||||
self,
|
||||
factor_df: pd.DataFrame,
|
||||
price_df: pd.DataFrame,
|
||||
fit: bool = True,
|
||||
) -> tuple[pd.DataFrame, pd.Series]:
|
||||
"""
|
||||
构建特征矩阵 X 和标签 y。
|
||||
|
||||
参数:
|
||||
factor_df: 因子 DataFrame, index=trade_date, columns=因子名
|
||||
price_df: 价格 DataFrame, 需有 'close'
|
||||
fit: True=训练模式（fit scaler + 记录有效特征），False=预测模式
|
||||
|
||||
返回:
|
||||
X, y（y 在 predict 模式下为 None）
|
||||
"""
|
||||
X = factor_df.copy()
|
||||
|
||||
# 1. 剔除 NaN 率过高的列
|
||||
if fit:
|
||||
nan_ratio = X.isna().mean()
|
||||
self._valid_features = list(nan_ratio[nan_ratio <= self.nan_threshold].index)
|
||||
# 排除非因子列
|
||||
self._valid_features = [c for c in self._valid_features
|
||||
if c not in ("close", "open", "high", "low", "volume")]
|
||||
X = X[self._valid_features].copy() if self._valid_features else X
|
||||
|
||||
# 2. 缺失值填充：前值填充 → 截面中位数
|
||||
X = X.ffill().fillna(X.median())
|
||||
|
||||
# 3. 去极值（Winsorize）
|
||||
if fit:
|
||||
lo, hi = self.winsorize_pct
|
||||
self._winsor_lower = X.quantile(lo)
|
||||
self._winsor_upper = X.quantile(hi)
|
||||
for col in X.columns:
|
||||
if col in getattr(self, "_winsor_lower", pd.Series()):
|
||||
X[col] = X[col].clip(self._winsor_lower[col], self._winsor_upper[col])
|
||||
|
||||
# 4. 标准化（训练时 fit，预测时 transform）
|
||||
if fit:
|
||||
X_scaled = self._scaler.fit_transform(X)
|
||||
self._scaler_fitted = True
|
||||
else:
|
||||
X_scaled = self._scaler.transform(X)
|
||||
|
||||
X = pd.DataFrame(X_scaled, index=X.index, columns=X.columns)
|
||||
|
||||
# 5. 构建标签
|
||||
y = self.build_labels(price_df) if fit else None
|
||||
|
||||
# 6. 对齐（删掉无法构建标签的行）
|
||||
if fit:
|
||||
valid_idx = X.index.intersection(y.dropna().index)
|
||||
X = X.loc[valid_idx]
|
||||
y = y.loc[valid_idx]
|
||||
|
||||
return X, y
|
||||
|
||||
# ── 多股票构建 ────────────────────────────────────────
|
||||
|
||||
def build_universe(
|
||||
self,
|
||||
factor_universe: dict[str, pd.DataFrame],
|
||||
price_universe: dict[str, pd.DataFrame],
|
||||
) -> tuple[pd.DataFrame, pd.Series]:
|
||||
"""多股票拼接特征矩阵（每只股票独立处理再拼接）。"""
|
||||
X_parts, y_parts = [], []
|
||||
for ts_code in factor_universe:
|
||||
f_df = factor_universe[ts_code]
|
||||
p_df = price_universe.get(ts_code)
|
||||
if p_df is None or f_df.empty or p_df.empty:
|
||||
continue
|
||||
X, y = self.build(f_df, p_df, fit=True)
|
||||
if X.empty:
|
||||
continue
|
||||
X["_ts_code"] = ts_code
|
||||
X_parts.append(X)
|
||||
y_parts.append(y)
|
||||
if not X_parts:
|
||||
return pd.DataFrame(), pd.Series()
|
||||
X_all = pd.concat(X_parts)
|
||||
y_all = pd.concat(y_parts)
|
||||
return X_all.drop(columns=["_ts_code"]), y_all
|
||||
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
LightGBM 模型封装。
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import lightgbm as lgb
|
||||
|
||||
from models.base import BaseModel
|
||||
from sklearn.model_selection import TimeSeriesSplit
|
||||
|
||||
_DEFAULT_PARAMS = {
|
||||
"objective": "regression",
|
||||
"metric": "rmse",
|
||||
"boosting_type": "gbdt",
|
||||
"num_leaves": 15,
|
||||
"learning_rate": 0.03,
|
||||
"feature_fraction": 0.7,
|
||||
"bagging_fraction": 0.7,
|
||||
"bagging_freq": 5,
|
||||
"verbose": -1,
|
||||
"n_estimators": 1000,
|
||||
"random_state": 42,
|
||||
"min_data_in_leaf": 20,
|
||||
}
|
||||
|
||||
|
||||
class LightGBMModel(BaseModel):
|
||||
"""LightGBM 回归模型。"""
|
||||
|
||||
name = "lightgbm"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
params: dict | None = None,
|
||||
early_stopping: int = 50,
|
||||
eval_ratio: float = 0.2,
|
||||
random_seed: int = 42,
|
||||
):
|
||||
self.params = params or _DEFAULT_PARAMS.copy()
|
||||
self.params["random_state"] = random_seed
|
||||
self.early_stopping = early_stopping
|
||||
self.eval_ratio = eval_ratio
|
||||
self._model: lgb.Booster | None = None
|
||||
self._feature_names: list[str] = []
|
||||
|
||||
def fit(self, X: pd.DataFrame, y: pd.Series) -> "LightGBMModel":
|
||||
self._feature_names = list(X.columns)
|
||||
|
||||
n = len(X)
|
||||
val_size = int(n * self.eval_ratio)
|
||||
|
||||
callbacks = []
|
||||
eval_set = None
|
||||
X_train, y_train = X, y
|
||||
|
||||
# 验证集足够大时才启用早停（至少 50 条）
|
||||
if val_size >= 50:
|
||||
split_idx = n - val_size
|
||||
X_train, X_val = X.iloc[:split_idx], X.iloc[split_idx:]
|
||||
y_train, y_val = y.iloc[:split_idx], y.iloc[split_idx:]
|
||||
eval_set = [(X_val, y_val)]
|
||||
callbacks = [
|
||||
lgb.early_stopping(stopping_rounds=self.early_stopping, verbose=False),
|
||||
lgb.log_evaluation(0),
|
||||
]
|
||||
|
||||
self._model = lgb.LGBMRegressor(**self.params)
|
||||
self._model.fit(
|
||||
X_train, y_train,
|
||||
eval_set=eval_set,
|
||||
callbacks=callbacks if callbacks else None,
|
||||
)
|
||||
return self
|
||||
|
||||
def predict(self, X: pd.DataFrame) -> pd.Series:
|
||||
if self._model is None:
|
||||
raise RuntimeError("模型尚未训练")
|
||||
preds = self._model.predict(X[self._feature_names])
|
||||
return pd.Series(preds, index=X.index, name="pred")
|
||||
|
||||
def get_feature_importance(self, importance_type: str = "gain") -> pd.DataFrame:
|
||||
"""特征重要性。importance_type: 'gain' | 'split'"""
|
||||
if self._model is None:
|
||||
return pd.DataFrame()
|
||||
imp = self._model.booster_.feature_importance(importance_type=importance_type)
|
||||
names = self._model.booster_.feature_name()
|
||||
df = pd.DataFrame({"feature": names, "importance": imp})
|
||||
df["importance_pct"] = df["importance"] / df["importance"].sum() * 100
|
||||
return df.sort_values("importance", ascending=False)
|
||||
|
||||
def cv_evaluate(
|
||||
self, X: pd.DataFrame, y: pd.Series, n_folds: int = 5
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
时间序列交叉验证评估（不 shuffle）。
|
||||
|
||||
返回每折的 IC (相关系数) 和 MSE。
|
||||
"""
|
||||
tscv = TimeSeriesSplit(n_splits=n_folds)
|
||||
results = []
|
||||
for fold, (train_idx, test_idx) in enumerate(tscv.split(X)):
|
||||
X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
|
||||
y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]
|
||||
|
||||
model = LightGBMModel(
|
||||
params=self.params,
|
||||
early_stopping=self.early_stopping,
|
||||
eval_ratio=0.0, # 不使用内部验证，直接全量训练
|
||||
)
|
||||
model.fit(X_train, y_train)
|
||||
preds = model.predict(X_test)
|
||||
ic = preds.corr(y_test)
|
||||
mse = ((preds - y_test) ** 2).mean()
|
||||
results.append({"fold": fold, "ic": round(ic, 4), "mse": round(mse, 4)})
|
||||
|
||||
df = pd.DataFrame(results)
|
||||
df.loc["mean"] = df.mean()
|
||||
return df
|
||||
|
||||
@property
|
||||
def n_estimators_used(self) -> int | None:
|
||||
"""实际使用的树数量（早停后可能 < n_estimators）。"""
|
||||
if self._model is None:
|
||||
return None
|
||||
return self._model.booster_.current_iteration()
|
||||
Reference in New Issue
Block a user