feat: 量化引擎加固 — 新增测试 + 数据/因子/回测层优化

- 新增 finance/tests/ 6 个测试套件（agents/backtest/dao_upsert/factors/features/fundamental_lookahead） - 数据层: data_manager / dao 优化，新增 upsert 逻辑 - 因子层: 基本面因子抽象定位 _mapping、ROE/PE/PB 重构 - 回测层: vectorbt/engine 大改动（251 行），report 增强 - ML 层: features/backtest_integration 特征工程与回测优化 - CLI: agent_cli 重构 - config/settings 扩充配置项
2026-08-31 14:01:06 +08:00
parent 6acf938caf
commit 73d191b43a
28 changed files with 1418 additions and 373 deletions
@@ -25,6 +25,7 @@ class BaseAgent(ABC):
            opt: OptunaEngine
            sent: SentimentEngine
            ml_models: dict[str, BaseModel]
+            feature_engine: FeatureEngine（已用模型训练集 fit 过，ML 打分需要）
        """
        self.dm = engines.get("dm")
        self.fe = engines.get("fe")
@@ -32,6 +33,7 @@ class BaseAgent(ABC):
        self.opt = engines.get("opt")
        self.sent = engines.get("sent")
        self.ml_models = engines.get("ml_models", {})
+        self.feature_engine = engines.get("feature_engine")

    @abstractmethod
    def execute(self, **kwargs) -> dict:
@@ -4,6 +4,7 @@ AgentOrchestrator — Agent 编排器。
 管理所有 Agent 的生命周期、执行顺序、结果传递。
 """

+import logging
 from datetime import datetime

 from agents.research_agent import ResearchAgent
@@ -11,6 +12,10 @@ from agents.selection_agent import SelectionAgent
 from agents.risk_agent import RiskAgent
 from agents.report_agent import ReportAgent

+logger = logging.getLogger(__name__)
+
+TOTAL_STEPS = 5
+

 class AgentOrchestrator:
    """
@@ -27,112 +32,153 @@ class AgentOrchestrator:
        self.agents: dict = {}
        self._last_results: dict = {}

+    def _log_step(self, current: int, title: str) -> None:
+        """统一步骤日志：分母固定为 TOTAL_STEPS。"""
+        print("\n[Step {}/{}] {}...".format(current, TOTAL_STEPS, title))
+
+    def _run_step(self, step_no, key, title, fn, results, on_ok=None):
+        """执行单个步骤并隔离异常。
+
+        key: 结果字典的键（英文，向前兼容如 risk/selection/report），
+        title: 步骤展示名（中文，仅用于日志）。
+        任一子步骤抛错都会被捕获：记录 results[key]={"error": ...},
+        让失败不中断整体流程；on_ok 用于成功后的副作用（打印等）。
+        """
+        self._log_step(step_no, title)
+        try:
+            value = fn()
+            results[key] = value
+            if on_ok:
+                on_ok(value)
+            return value
+        except Exception as e:
+            logger.exception("[Orchestrator] 步骤 %s 失败: %s", title, e)
+            results[key] = {"error": str(e)}
+            print("  [SKIP] 步骤 {} 失败，继续后续步骤: {}".format(title, e))
+            return None
+
    def setup(self):
        """注册所有 Agent。"""
        self.agents["research"] = ResearchAgent(**self.engines)
        self.agents["selection"] = SelectionAgent(**self.engines)
        self.agents["risk"] = RiskAgent(**self.engines)
        self.agents["report"] = ReportAgent(**self.engines)
-        print(f"[Orchestrator] 已注册 {len(self.agents)} 个 Agent: {list(self.agents)}")
+        logger.info("[Orchestrator] 已注册 %s 个 Agent: %s",
+                    len(self.agents), list(self.agents))

    # ── 每日流程 ──────────────────────────────────────────

    def run_daily(self, date: str | None = None) -> dict:
        """
-        每日任务流:
+        每日任务流（5 步，任一步失败不中断整体）:

-        1. 同步数据
+        1. 同步数据（增量同步已缓存股票）
        2. 风险评估
        3. 股票打分
-        4. 生成日报
+        4. 情绪因子
+        5. 生成日报
        """
        date = date or datetime.now().strftime("%Y%m%d")
        print(f"\n{'='*60}")
-        print(f"[Orchestrator] 每日流程 — {date}")
+        logger.info("[Orchestrator] 每日流程 — %s", date)
        print(f"{'='*60}")

-        results = {"date": date}
+        results: dict = {"date": date}

        # Step 1: 增量同步已缓存股票的最新行情
-        print("\n[Step 1/4] 同步行情...")
        dm = self.engines.get("dm")
        sent = self.engines.get("sent")
        from database.dao import get_latest_trade_date

-        scope_stocks = []
-        if sent:
-            scope_stocks = sent.get_scope_stocks()
-        if not scope_stocks and dm:
-            scope_stocks = list(dm.get_stock_list().index[:100])
-        print("  范围: {} 只股票".format(len(scope_stocks)))
+        def _step_sync():
+            scope_stocks = []
+            if sent:
+                scope_stocks = sent.get_scope_stocks()
+            if not scope_stocks and dm:
+                scope_stocks = list(dm.get_stock_list().index[:100])
+            print("  范围: {} 只股票".format(len(scope_stocks)))

-        # 分类：已缓存（增量更新），未缓存（统计跳过）
-        cached = [c for c in scope_stocks if get_latest_trade_date(c)]
-        uncached = len(scope_stocks) - len(cached)
-        print("  已缓存: {} 只 (增量更新), 未缓存: {} 只 (跳过，需首次批量预热)".format(len(cached), uncached))
+            cached = [c for c in scope_stocks if get_latest_trade_date(c)]
+            uncached = len(scope_stocks) - len(cached)
+            print("  已缓存: {} 只 (增量更新), 未缓存: {} 只 (需首次预热)".format(len(cached), uncached))

-        synced = 0
-        for i, ts_code in enumerate(cached):
-            try:
-                n = dm.sync_daily(ts_code)
-                synced += n
-            except Exception:
-                continue
-            if (i + 1) % 100 == 0:
-                print("  [同步] 进度: {}/{}".format(i + 1, len(cached)))
-        print("  同步完成: {} 条新数据 (已缓存{}/全量{})".format(synced, len(cached), len(scope_stocks)))
+            synced = 0
+            for i, ts_code in enumerate(cached):
+                try:
+                    n = dm.sync_daily(ts_code)
+                    synced += n
+                except Exception as e:
+                    logger.debug("同步 %s 失败: %s", ts_code, e)
+                    continue
+                if (i + 1) % 100 == 0:
+                    logger.info("[同步] 进度: %s/%s", i + 1, len(cached))
+            if uncached > 0:
+                print("  提示: {} 只股票未缓存，运行 'agent_cli.py warmup' 首次批量预热".format(uncached))
+            return {"synced": synced, "cached": len(cached), "uncached": uncached,
+                    "universe_size": len(scope_stocks), "scope_stocks": scope_stocks}

-        if uncached > 0:
-            print("  提示: {} 只股票未缓存，运行 'agent_cli.py warmup' 首次批量预热".format(uncached))
+        sync_info = self._run_step(1, "sync", "同步行情", _step_sync, results, on_ok=lambda r: print(
+            "  同步完成: {} 条新数据 (已缓存{}/全量{})".format(
+                r["synced"], r["cached"], r["universe_size"])))

        # Step 2: 风险评估
-        print("\n[Step 2/4] 风险评估...")
-        risk = self.agents["risk"].execute()
-        results["risk"] = risk
-        print(f"  风险: {risk['risk_level']}, 仓位: {risk['target_exposure']:.0%}")
+        def _risk():
+            risk = self.agents["risk"].execute()
+            risk_level = risk.get("risk_level", "?")
+            exposure = risk.get("target_exposure", 0)
+            print("  风险: {}, 仓位: {:.0%}".format(risk_level, exposure))
+            return risk
+        risk = self._run_step(2, "risk", "风险评估", _risk, results)

        # Step 3: 选股打分
-        print("\n[Step 3/4] 股票打分...")
-        selection = self.agents["selection"].execute(date=date, top_n=15)
-        results["selection"] = selection
-        top = selection.get("top_picks", [])
-        if top:
-            print("  Top 5: {}".format(", ".join(p['ts_code'] for p in top[:5])))
+        def _selection():
+            selection = self.agents["selection"].execute(date=date, top_n=15)
+            top = selection.get("top_picks", [])
+            if top:
+                print("  Top 5: {}".format(", ".join(p['ts_code'] for p in top[:5])))
+            return selection
+        selection = self._run_step(3, "selection", "股票打分", _selection, results)

        # Step 4: 情绪因子
-        print("\n[Step 4/5] 情绪因子...")
-        sentiment_df = None
-        sent_eng = self.engines.get("sent")
-        if sent_eng:
-            try:
-                # 用范围内第一只有缓存的股票计算情绪因子
-                ref_code = cached[0] if cached else "000001.SZ"
-                sentiment_df = sent_eng.compute(ref_code, max_news=30)
-                if sentiment_df is not None and not sentiment_df.empty:
-                    valid = sentiment_df.dropna(how="all")
-                    print("  {}: {} 个因子, {} 个有效交易日".format(ref_code, sentiment_df.shape[1], len(valid)))
-                else:
-                    print("  (无有效情绪数据)")
-            except Exception as e:
-                print("  [SKIP] 情绪因子计算失败: {}".format(e))
-        else:
-            print("  (SentimentEngine 未配置)")
-        results["sentiment"] = sentiment_df
+        def _sentiment():
+            sentiment_df = None
+            sent_eng = self.engines.get("sent")
+            if not sent_eng:
+                print("  (SentimentEngine 未配置)")
+                return None
+            # 用范围内第一只有缓存的股票计算情绪因子；无缓存信息时回退沪市指数代码
+            scope = (sync_info or {}).get("scope_stocks") or []
+            ref_code = scope[0] if scope else "000001.SZ"
+            sentiment_df = sent_eng.compute(ref_code, max_news=30)
+            if sentiment_df is not None and not sentiment_df.empty:
+                valid = sentiment_df.dropna(how="all")
+                print("  {}: {} 个因子, {} 个有效交易日".format(
+                    ref_code, sentiment_df.shape[1], len(valid)))
+            else:
+                print("  (无有效情绪数据)")
+            return sentiment_df
+        sentiment_df = self._run_step(4, "sentiment", "情绪因子", _sentiment, results)

        # Step 5: 生成日报
-        print("\n[Step 5/5] 生成日报...")
-        report = self.agents["report"].execute(
-            date=date,
-            selection_result=selection,
-            risk_result=risk,
-            sentiment_result=sentiment_df,
-        )
-        results["report"] = report
-        print(f"  日报: {report['report_path']}")
+        def _report():
+            rep = self.agents["report"].execute(
+                date=date,
+                selection_result=selection,
+                risk_result=risk,
+                sentiment_result=sentiment_df,
+            )
+            print("  日报: {}".format(rep.get("report_path", "?")))
+            return rep
+        report = self._run_step(5, "report", "生成日报", _report, results)

        self._last_results = results
        print(f"\n{'='*60}")
-        print(f"[Orchestrator] 每日流程完成")
+        print("[Orchestrator] 每日流程完成")
+        failed = [k for k, v in results.items() if isinstance(v, dict) and "error" in v]
+        if failed:
+            print("[Orchestrator] 有步骤未完成: {}".format(", ".join(failed)))
+        else:
+            print("[Orchestrator] 所有步骤均成功")
        print(f"{'='*60}\n")
        return results

@@ -140,16 +186,15 @@ class AgentOrchestrator:

    def run_research_cycle(self, ts_codes: list[str] | None = None) -> dict:
        """
-        研究周期:
+        研究周期（1 步）:

-        1. 因子发现与评估
-        2. 更新 IC 权重
+        1. 因子发现与评估（IC / IC_IR / 分层收益）
        """
        print(f"\n{'='*60}")
-        print(f"[Orchestrator] 研究周期")
+        logger.info("[Orchestrator] 研究周期")
        print(f"{'='*60}")

-        print("\n[Step 1/2] 因子发现...")
+        print("\n[Step 1/1] 因子发现...")
        research = self.agents["research"].execute(ts_codes=ts_codes)
        results = {"research": research}

@@ -8,6 +8,9 @@ import numpy as np
 import pandas as pd

 from agents.base import BaseAgent
+from config.settings import (
+    RISK_THRESHOLDS, RISK_EXPOSURE, RISK_MAX_SINGLE, RISK_STOP_LOSS,
+)


 class RiskAgent(BaseAgent):
@@ -16,11 +19,8 @@ class RiskAgent(BaseAgent):
    name = "Risk"
    description = "仓位控制与风险预警"

-    # 风险等级阈值
-    THRESHOLDS = {
-        "high": {"vol": 35, "dd": -15},
-        "medium": {"vol": 25, "dd": -8},
-    }
+    # 风险等级阈值（来自配置中心）
+    THRESHOLDS = RISK_THRESHOLDS

    def execute(
        self,
@@ -55,24 +55,22 @@ class RiskAgent(BaseAgent):
        peak = close.expanding().max()
        current_dd = float((close.iloc[-1] / peak.iloc[-1] - 1) * 100)

-        # 风险等级
+        # 风险等级 & 仓位（参数来自配置中心）
        if market_vol > self.THRESHOLDS["high"]["vol"] or current_dd < self.THRESHOLDS["high"]["dd"]:
            risk_level = "high"
-            target_exposure = 0.30
        elif market_vol > self.THRESHOLDS["medium"]["vol"] or current_dd < self.THRESHOLDS["medium"]["dd"]:
            risk_level = "medium"
-            target_exposure = 0.60
        else:
            risk_level = "low"
-            target_exposure = 0.85
+        target_exposure = RISK_EXPOSURE[risk_level]

        # 最近 N 日涨跌
        ret_5d = float(close.pct_change(5).iloc[-1] * 100) if len(close) >= 6 else 0
        ret_20d = float(close.pct_change(20).iloc[-1] * 100) if len(close) >= 21 else 0

-        # 单票上限（风险越高越集中）
-        max_single = 0.15 if risk_level == "low" else (0.10 if risk_level == "medium" else 0.05)
-        stop_loss = -0.05 if risk_level == "low" else (-0.08 if risk_level == "medium" else -0.12)
+        # 单票上限（风险越高越集中）与止损线
+        max_single = RISK_MAX_SINGLE[risk_level]
+        stop_loss = RISK_STOP_LOSS[risk_level]

        # 持仓预警
        alerts = []
@@ -8,6 +8,10 @@ import numpy as np
 import pandas as pd

 from agents.base import BaseAgent
+from config.settings import (
+    SELECTION_CORE_FACTORS, SELECTION_UNIVERSE_SIZE,
+    SELECTION_SCORE_LIMIT, SELECTION_WINSORIZE_ZSCORE,
+)


 class SelectionAgent(BaseAgent):
@@ -43,18 +47,13 @@ class SelectionAgent(BaseAgent):
                ts_codes = self.sent.get_scope_stocks()
            else:
                stocks = self.dm.get_stock_list()
-                ts_codes = list(stocks.index[:100])
+                ts_codes = list(stocks.index[:SELECTION_UNIVERSE_SIZE])

        if not ts_codes:
            return {"date": date or self._today(), "top_picks": [], "score_df": pd.DataFrame()}

-        # 选择核心因子（覆盖多个维度，减少计算量）
-        core_factors = [
-            "momentum_20", "momentum_60",
-            "rsi_14", "volatility_20",
-            "vol_ratio_5", "ma_dev_20",
-            "turnover_5", "amplitude_5",
-        ]
+        # 选择核心因子（覆盖多个维度，减少计算量；参数来自配置中心）
+        core_factors = list(SELECTION_CORE_FACTORS)
        factor_objects = [get_factor(n) for n in core_factors]

        self.log("打分 {} 只股票 (权重={})".format(len(ts_codes), weighting))
@@ -70,7 +69,7 @@ class SelectionAgent(BaseAgent):
            len(available) / len(ts_codes) * 100 if ts_codes else 0))

        # 2. 打分（限制上限防止单次太慢）
-        score_limit = min(len(available), 300)
+        score_limit = min(len(available), SELECTION_SCORE_LIMIT)
        scores = {}
        valid_count = 0
        for i, ts_code in enumerate(available[:score_limit]):
@@ -165,23 +164,38 @@ class SelectionAgent(BaseAgent):
        if len(row_clean) < 3:
            return None

-        # z-score 标准化
-        z = (row_clean - factor_df[row_clean.index].mean()) / factor_df[row_clean.index].std().replace(0, 1)
+        # z-score 标准化（用历史均值/标准差），可选的去极值避免单股离群主导 Top
+        cols = row_clean.index
+        mu = factor_df[cols].mean()
+        std = factor_df[cols].std().replace(0, 1)
+        z = (row_clean - mu) / std
+        if SELECTION_WINSORIZE_ZSCORE:
+            z = z.clip(-3, 3)
        return float(z.mean())

    def _score_ml(self, ts_code: str, factor_df: pd.DataFrame, date: str | None) -> float | None:
-        """ML 模型打分。"""
-        from models.features import FeatureEngine
-        fe = FeatureEngine(lookahead=5)
+        """ML 模型打分。

-        daily = self.dm.get_daily(ts_code)
+        要求注入名为 feature_engine 的、已用训练集 fit 过的 FeatureEngine，
+        以及 ml_models（已训练模型）。两者缺一时明确退出，而不是沿用旧的
+        未 fit 引擎静默失败。
+        """
+        fe = self.feature_engine
+        if fe is None:
+            self.log("ML 打分需要注入 feature_engine（已 fit），当前未提供，跳过 ML 打分")
+            return None
+        if not self.ml_models:
+            self.log("ML 打分需要 ml_models（已训练），当前为空，跳过 ML 打分")
+            return None
+
+        daily = self.dm.get_daily(ts_code) if self.dm else None
        if daily is None or daily.empty:
            return None
        daily = daily.set_index("trade_date")

        try:
            X, _ = fe.build(factor_df, daily, fit=False)
-            if X.empty:
+            if X is None or X.empty:
                return None
            if date and date in X.index:
                X = X.loc[[date]]
@@ -191,5 +205,7 @@ class SelectionAgent(BaseAgent):
            model = self.ml_models.get("lightgbm") or list(self.ml_models.values())[0]
            pred = model.predict(X)
            return float(pred.iloc[0]) if len(pred) > 0 else None
-        except Exception:
+        except Exception as e:
+            # 不再静默返回 None：记录原因，便于定位预测路径问题
+            self.log("[WARN] ML 打分失败 ({}): {}".format(ts_code, e))
            return None