diff --git a/.claude/skills/batch-sync.md b/.claude/skills/batch-sync.md deleted file mode 100644 index 36cb60d..0000000 --- a/.claude/skills/batch-sync.md +++ /dev/null @@ -1,26 +0,0 @@ -# batch-sync skill - -批量预热股票数据到 DB 缓存。 - -## 触发 - -用户说:预热缓存 / 同步数据 / warmup / batch sync / 补齐数据 / 全量同步 - -## 执行 - -```bash -cd finance && python cli/agent_cli.py warmup 50 -``` - -## 说明 - -- 每批 50 只股票,依次执行 `dm.sync_daily()` -- 已缓存 + 最新的 → 0 条跳过(增量) -- 未缓存 → Tushare 优先 → AkShare fallback -- 范围由 `.env` 中 `SENTIMENT_SCOPE_TYPE` + `SENTIMENT_SCOPE_INDEXES` 决定(默认沪深300+中证500) -- 可多次执行直到覆盖率 100% - -## 参数 - -`python cli/agent_cli.py warmup [N]` -- N: 每批股票数,默认 50。网络稳定时可调大到 100 diff --git a/README.md b/README.md index cc1e573..7b4a95a 100644 --- a/README.md +++ b/README.md @@ -7,225 +7,123 @@ ``` cc-cursor/ ├── finance/ # 核心量化引擎 -│ ├── config/ # 全局配置(MariaDB / AkShare) -│ ├── database/ # ORM 模型 + DAO(mac_ 前缀表) +│ ├── config/ # 全局配置 +│ ├── database/ # ORM 模型 + DAO │ ├── data/ # DataManager 统一数据层 -│ ├── factors/ # 因子引擎(34 因子 / 12 分类,含情绪因子) -│ ├── backtest/ # 回测引擎(VectorBT + 5 策略 + 截面回测) -│ ├── optimizer/ # Optuna 参数优化 + Walk-Forward +│ ├── factors/ # 因子引擎(34 因子 / 12 分类) +│ ├── backtest/ # 回测引擎(VectorBT + 5 策略) +│ ├── optimizer/ # Optuna 参数优化 │ ├── models/ # LightGBM / CatBoost ML 模型 -│ ├── agents/ # Agent 系统(4 Agent + 编排器 + CLI) +│ ├── agents/ # Agent 系统(4 Agent + 编排器) │ ├── cli/ # 命令行 & 验证脚本 -│ ├── reports/ # 自动日报输出目录 -│ └── .env # 环境变量配置(API Key / 分析范围) -├── djapi/ # Django API 后端(A 股数据 + 新闻联播 + 日报查询) -├── mcp-servers/ # MCP Server(Serena,本机工具,git 忽略) -├── shared/ # 共享工具(SSH 隧道脚本) -├── docs/ # 文档 & 使用指南 -└── .claude/ # Claude Code 配置 +│ └── reports/ # 日报输出目录 +├── djapi/ # Django API 后端 +├── shared/script/ # SSH 隧道脚本 +└── docs/ # 项目文档 ``` ## 数据流 ``` -Agent 编排层 - ├── ResearchAgent ── 因子发现(IC/IC_IR 评估) - ├── SelectionAgent ─ 多因子打分 + ML 预测 - ├── RiskAgent ────── 仓位控制 + 风险预警 - └── ReportAgent ──── 自动日报生成 - -基础引擎层 - DataManager ──→ FactorEngine ──→ BaseStrategy ──→ VectorBTEngine ──→ BacktestReport - │ │ │ - │ FeatureEngine OptunaEngine - │ │ │ - └──────→ LightGBM/CatBoost ←────────┘ - -情绪增强层 - NewsSource(AkShare/DB/MCP) ──→ QwenClient ──→ SentimentFactor ──→ FactorEngine +Data → Factor → Model → Strategy → Backtest → Report ``` -全部通过 Service 层中转:策略不直连 AkShare,模型不直连数据库,Agent 不重建引擎。 - ---- - -## 开发进度 - -| Sprint | 模块 | 关键成果 | 状态 | -|--------|------|----------|------| -| Sprint 0 | 基础设施 | DataManager + MariaDB 3 表 | ✅ | -| Sprint 1 | 因子引擎 | 34 因子 / 12 分类 | ✅ | -| Sprint 2 | 回测引擎 | VectorBT + 5 策略 + 截面回测 | ✅ | -| Sprint 3 | 参数优化 | Optuna + Walk-Forward | ✅ | -| Sprint 4 | ML 模型 | LightGBM + CatBoost + 特征工程 | ✅ | -| Sprint 5 | 情绪因子 | Qwen + 三源新闻聚合 + 日期对齐 | ✅ | -| Sprint 6 | Agent 系统 | 4 Agent + 编排器 + CLI + 自动日报 | ✅ | -| Sprint 7 | djapi API | 日报查询 ×2(news/reports + news/events) | ✅ | - -**全部 8 个 Sprint 已完成。** - ---- - -## 功能模块 - -### 数据层 `finance/data/` - -```python -from data.data_manager import DataManager -dm = DataManager(); dm.init_db() -stocks = dm.get_stock_list() # → 5,524 只 -daily = dm.get_daily("000001.SZ") # → 日线 -fina = dm.get_financial("000001.SZ") # → 财务数据 -dm.sync_daily("000001.SZ") # → 增量同步 -``` - -### 因子引擎 `finance/factors/` - -```python -from factors.registry import get_factor, list_factors -from factors.engine import FactorEngine - -engine = FactorEngine(dm) -factors = [get_factor("momentum_20"), get_factor("rsi_14")] -factor_df = engine.compute("000001.SZ", factors) -# → 34 个注册因子,12 个分类(动量/RSI/MACD/量价/布林/ATR/均线/波动率/换手率/振幅/基本面/情绪) -``` - -### 回测引擎 `finance/backtest/` - -```python -from backtest.vectorbt.engine import VectorBTEngine -from backtest.strategies.rsi_mean_revert import RSIMeanRevertStrategy - -engine_bt = VectorBTEngine(initial_capital=100_000, commission=0.0003) -report = engine_bt.run(RSIMeanRevertStrategy(oversold=30, overbought=70), price_df, factor_df) -# → 收益=29.4% 年化=4.3% 回撤=-19.1% 夏普=0.37 胜率=77.1% -``` - -5 个内置策略 + 自定义策略接口 + 截面回测 + BacktestReport 标准化报告。 - -### 参数优化 `finance/optimizer/` - -```python -from optimizer.engine import OptunaEngine -from optimizer.space import rsi_revert_space - -result = OptunaEngine(engine_bt).optimize( - RSIMeanRevertStrategy, rsi_revert_space, price_df, factor_df, - metric="sharpe", n_trials=200, -) -# → 最优参数: oversold=13, overbought=66 -# → 夏普: 0.37→0.60 (+62%), 回撤: -19.1%→-1.8% (10倍改善) -``` - -7 种优化目标 + 4 个预置搜索空间 + Walk-Forward 滚动验证 + 快捷函数。 - -### ML 模型 `finance/models/` - -```python -from models.features import FeatureEngine -from models.lightgbm.model import LightGBMModel - -fe = FeatureEngine(lookahead=5) -X, y = fe.build(factor_df, price_df, fit=True) -model = LightGBMModel(params={"n_estimators": 200}).fit(X_train, y_train) -pred = model.predict(X_test) # → IC 评估 + 特征重要性 + 交叉验证 + ML 策略回测 -``` - -Winsorize → 缺失填充 → RobustScaler → LightGBM/CatBoost 训练 → MLBenchmark 对比。 - -### 情绪因子 `finance/factors/sentiment/` - -```python -from factors.sentiment.sentiment_engine import SentimentEngine - -sent = SentimentEngine(dm) -sent_df = sent.compute("000001.SZ", max_news=20) -# → news_sent_5, news_conf_5, sent_delta_5 -``` - -三数据源聚合(AkShare 个股新闻 + 新闻联播 DB + MCP trendradar-news)、日期对齐(非交易日→最近交易日)、xwlb 偏移(昨日新闻→今日使用)、DashScope + Ollama 双后端。 - -### Agent 系统 `finance/agents/` - -```bash -python finance/cli/agent_cli.py daily # 完整每日流程 -python finance/cli/agent_cli.py picks 15 # 选股 Top 15 -python finance/cli/agent_cli.py risk # 风险评估 -python finance/cli/agent_cli.py research # 因子研究 -python finance/cli/agent_cli.py report # 生成日报 -``` - -4 个 Agent(Research/Selection/Risk/Report)+ 编排器 + 自动日报(reports/daily_YYYYMMDD.md)。 - ---- - ## 快速开始 ```bash -# SSH 隧道 +# 环境 & SSH 隧道 +conda activate quant bash shared/script/autossh.sh -# Python 环境 -conda activate quant # Python 3.11.13 - # 每日 Agent 运行 python finance/cli/agent_cli.py daily ``` -### 验证脚本 +## CLI 命令 ```bash -python finance/cli/demo_data_manager.py # Sprint 0 — DataManager -python finance/cli/demo_factor_engine.py # Sprint 1 — 因子引擎 -python finance/cli/demo_backtest.py # Sprint 2 — 回测引擎 -python finance/cli/demo_optimizer.py # Sprint 3 — 参数优化 -python finance/cli/demo_ml.py # Sprint 4 — ML 模型 -python finance/cli/demo_sentiment.py # Sprint 5 — 情绪因子 -python finance/cli/demo_sentiment_detail.py # Sprint 5 — 情绪因子(单股详情) +python finance/cli/agent_cli.py daily # 5 步完整流程 +python finance/cli/agent_cli.py picks 15 # 选股 Top 15 +python finance/cli/agent_cli.py risk # 风险评估 +python finance/cli/agent_cli.py research # 因子研究 +python finance/cli/agent_cli.py report 20260603 # 生成日报 +python finance/cli/agent_cli.py warmup 50 # 首次预热缓存 ``` ---- +## 验证脚本 + +```bash +python finance/cli/demo_data_manager.py --ts_code 600519.SH +python finance/cli/demo_factor_engine.py --ts_code 300750.SZ +python finance/cli/demo_backtest.py --ts_code 000001.SZ +python finance/cli/demo_optimizer.py --ts_code 000001.SZ --trials 100 +python finance/cli/demo_ml.py --ts_code 000001.SZ --lookahead 5 +python finance/cli/demo_sentiment.py --ts_code 600519.SH +python finance/cli/demo_sentiment_detail.py --ts_code 600519.SH --date 20260603 +``` ## 技术栈 -| 组件 | 技术 | 版本 | 状态 | -|------|------|------|------| -| 数据获取 | AkShare | 1.18.64 | ✅ | -| 数据库 | MariaDB (SSH 隧道) | — | ✅ | -| 因子/特征 | pandas / numpy / sklearn | 2.3 / 2.0 / 1.9 | ✅ | -| 回测引擎 | VectorBT | 1.0 | ✅ | -| 参数优化 | Optuna | 4.9 | ✅ | -| ML 模型 | LightGBM / CatBoost | 4.6 / 1.2 | ✅ | -| NLP 情绪 | Qwen (DashScope / Ollama) | turbo / 2.5 | ✅ | -| Agent 框架 | 自研编排器 | — | ✅ | -| API 后端 | Django + uWSGI | 5.2 | 已有 | -| 代码分析 | Serena MCP | — | 本机工具(不随仓库分发) | +| 组件 | 技术 | +|------|------| +| 数据获取 | AkShare + Tushare (双源) | +| 数据库 | MariaDB (SSH 隧道) | +| 因子/特征 | pandas / numpy / sklearn | +| 回测引擎 | VectorBT 1.0 | +| 参数优化 | Optuna 4.9 | +| ML 模型 | LightGBM 4.6 + CatBoost 1.2 | +| NLP 情绪 | Qwen (DashScope / Ollama) | +| Agent 编排 | 自研编排器 | +| API 后端 | Django 5.2 + uWSGI | ---- +## 开发进度 -## 设计原则 +| Sprint | 模块 | 状态 | +|--------|------|------| +| Sprint 0 | 基础设施(DataManager + MariaDB) | ✅ | +| Sprint 1 | 因子引擎(34 因子 / 12 分类) | ✅ | +| Sprint 2 | VectorBT 回测(5 策略 + 截面) | ✅ | +| Sprint 3 | Optuna 优化(+ Walk-Forward) | ✅ | +| Sprint 4 | ML 模型(LightGBM + CatBoost) | ✅ | +| Sprint 5 | Qwen 情绪因子(三源新闻) | ✅ | +| Sprint 6 | Agent 系统(4 Agent + CLI) | ✅ | +| Sprint 7 | djapi API(日报查询 ×2) | ✅ | -- **模块隔离**:各引擎通过统一接口交互,可替换实现(VectorBT → Backtrader) -- **接口标准化**:因子 `calculate(df)→Series` / 策略 `generate_signals(df)→Series` / 模型 `fit/predict/save/load` / 优化 `optimize()→Result` -- **数据层统一**:策略/模型不直连数据源,全部通过 DataManager -- **Agent 不重建轮子**:Agent 通过依赖注入复用已有引擎,编排而非重建 -- **防前视偏差**:时间序列交叉验证、expanding window 统计量 -- **渐进演进**:全链路 8 个 Sprint 平滑推进,无推倒重写 +**全部 8 个 Sprint 已完成。** ## 文档 -- [使用指南](./docs/usage.md) — 详细使用说明(12 章节,含代码示例) -- [使用指南 (HTML)](./docs/usage.html) — 网页版使用指南 -- [新闻日报 API](./docs/news_report_api.md) — djapi 日报查询接口使用手册 +| 文档 | 内容 | +|------|------| +| [使用指南](docs/usage.md) | 各模块使用方法和代码示例 | +| [架构说明](docs/architecture.md) | 项目架构、数据流、设计原则 | +| [开发指南](docs/development.md) | 环境搭建、开发约定、模块说明 | +| [部署说明](docs/deployment.md) | 本地环境、服务器、uWSGI、rsync 部署 | +| [因子与表结构速查](docs/reference.md) | 34 因子注册表、DB 表结构 | +| [数据层详解](docs/data-layer.md) | DataManager、数据库、缓存策略 | +| [因子引擎详解](docs/factors.md) | 因子计算、情绪引擎、新闻源 | +| [回测引擎详解](docs/backtest.md) | VectorBT、策略、信号工具、Optuna | +| [ML 模型详解](docs/ml-models.md) | 特征工程、LightGBM/CatBoost | +| [Agent 系统详解](docs/agents.md) | Agent 架构、CLI、日报 | +| [DJAPI 接口](docs/api.md) | Django API 端点参考 | +| [日报查询 API](docs/news_report_api.md) | news/reports + news/events 接口 | +| [日报数据库](docs/db_schema_v1.1.md) | news_report / news_event 表结构 | ## 子项目 -- [djapi](./djapi/README.md) — Django API 后端:A 股数据 API(16 端点)+ 新闻联播处理 + 日报查询(news/reports、news/events) +- [djapi](djapi/README.md) — Django API 后端:A 股数据 API(16 端点)+ 新闻联播处理 + 日报查询 + +## 设计原则 + +- **模块隔离**:各引擎通过统一接口交互,可替换实现 +- **接口标准化**:因子 `calculate(df)→Series` / 策略 `generate_signals(df)→Series` / 模型 `fit/predict/save/load` +- **数据层统一**:策略/模型不直连数据源,全部通过 DataManager +- **Agent 不重建轮子**:Agent 通过依赖注入复用已有引擎 +- **防前视偏差**:时间序列交叉验证、expanding window 统计量 ## 数据库连接 ```bash bash shared/script/autossh.sh # host: 127.0.0.1:13306 user: myquant database: myquant table_prefix: mac_ -``` +``` \ No newline at end of file diff --git a/continuation.md b/continuation.md deleted file mode 100644 index 1d76a09..0000000 --- a/continuation.md +++ /dev/null @@ -1,83 +0,0 @@ -# continuation.md — cc-cursor 项目状态 - -生成时间:2026-06-07(全部 Sprint 完成 + 生产加固 + djapi 数据源归一化 + Git 初始化) - ---- - -## Git 状态 - -- 仓库:https://github.com/Simon2046/myquant -- 分支:`main` -- commit:`271a934` — Initial commit: cc-cursor 全链路量化研究平台 -- 文件:293 个文件,59,598 行 -- 已排除:`.env`、`mcp-servers/serena`、`__pycache__`、`.parquet`、`.db` - ---- - -## 全部 Sprint 完成 ✅ - -| Sprint | 模块 | 状态 | -|--------|------|------| -| 0 | 基础设施(DataManager + MariaDB) | ✅ | -| 1 | 因子引擎(34 因子 / 12 分类) | ✅ | -| 2 | VectorBT 回测(5 策略 + 截面 + BacktestReport) | ✅ | -| 3 | Optuna 优化(+ Walk-Forward) | ✅ | -| 4 | ML 模型(LightGBM + CatBoost + MLStrategy) | ✅ | -| 5 | Qwen 情绪因子(三源新闻 + 日期对齐) | ✅ | -| 6 | Agent 系统(4 Agent + CLI + 日报 .md/.html) | ✅ | - ---- - -## 生产稳定性加固(14 项) - -| # | 项 | 文件 | -|---|-----|------| -| 1 | Tushare 双数据源(优先) | `finance/data/data_manager.py` | -| 2 | 指数 vs 个股自动路由 | `finance/data/sources/akshare_source.py` | -| 3 | SSH 自动恢复(多次重连 + pool_pre_ping) | `finance/database/connection.py` | -| 4 | save_daily 先删后插(防主键冲突) | `finance/database/dao.py` | -| 5 | load_dotenv 绝对路径 + 模块加固 | `finance/config/settings.py` + 3 文件 | -| 6 | 日报 5d/20d 修复(idx=-1→pos=len-1) | `finance/agents/report_agent.py` | -| 7 | RiskAgent 改用上证指数 | `finance/agents/risk_agent.py` | -| 8 | 日报增加"昨日对比" + 数据截止 | `finance/agents/report_agent.py` | -| 9 | mac_report 表 utf8mb4 + DATE + DATETIME | `finance/database/models.py` | -| 10 | 日报自动存入 DB + emoji 兼容 | `finance/reports/storage.py` + 8 CLI | -| 11 | CLAUDE-*.md 文档化 9 条已知 Bug | `CLAUDE-data.md` + `CLAUDE-agents.md` | -| 12 | demo 脚本全参数化 | `finance/cli/demo_*.py` | -| 13 | **djapi 数据源归一化(10→1 入口)** | `djapi/api/stock/data_source.py` | -| 14 | **indexDatas API 参数修正 + 容错** | `djapi/api/views.py` + `getIndexs.py` | - ---- - -## djapi 数据源归一化 - -- 新增 `djapi/api/stock/data_source.py` — 统一入口 - - `get_tushare_pro()` — 全局单例(线程安全) - - `get_daily()` — 双源 fallback (Tushare→AkShare) - - `get_mysql_db()` — MySQL 全局单例 - - Token 兼容 `TUSHARE_TS_TOKEN` / `TUSHARE_TOKEN` -- 10 个模块迁移完成 -- `getDivData_AK.py` 标记废弃 -- `getIndexs.py` 修复:`index_dailybasic` 失败不阻塞,异常 raise 而非静默返回空 -- `views.py` 修正:`indexDatas` 参数 `index_name` → `tscode`,描述从"股票代码"→"指数代码",新增 `_PARAM_INDEX_CODE` -- 已部署到 `api.doorcome.cn` ✅ - ---- - -## CLI 命令 - -```bash -agent_cli.py daily / picks / risk / research / report / warmup -demo_*.py(全部支持 --ts_code --date 等参数) -``` - -## 文档 - -- `CLAUDE.md` — 入口 + 路由 + 多步任务规则 -- `CLAUDE-data.md` — 数据层 + 5 条已知 Bug -- `CLAUDE-factors.md` — 因子引擎 -- `CLAUDE-backtest.md` — 回测 + 优化 -- `CLAUDE-ml.md` — ML 模型 -- `CLAUDE-agents.md` — Agent + CLI + 4 条已知 Bug -- `CLAUDE-reference.md` — 因子/表结构速查 -- `docs/usage.md` + `docs/usage.html` — 使用指南 diff --git a/djapi/.env.example b/djapi/.env.example index 6c373a7..44b573a 100644 --- a/djapi/.env.example +++ b/djapi/.env.example @@ -13,7 +13,7 @@ MYSQL_PASSWORD=your-mysql-password MYSQL_DATABASE=myquant # 日报结构化入库 (news_report / news_event) 只读查询 -# 与 report_db_design.md §7 一致;密码必填,缺失时接口直接报错 +# 与 docs/db_schema_v1.1.md 一致;密码必填,缺失时接口直接报错 NEWS_DB_HOST=127.0.0.1 NEWS_DB_PORT=3306 NEWS_DB_USER=myquant diff --git a/djapi/.mcp.json b/djapi/.mcp.json deleted file mode 100644 index b652839..0000000 --- a/djapi/.mcp.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "mcpServers": { - "serena-djapi": { - "command": "uv", - "args": [ - "run", - "--directory", - "/Users/summer/Downloads/cc-cursor/mcp-servers/serena", - "serena", - "start-mcp-server", - "--project", - "/Users/summer/Downloads/cc-cursor/djapi" - ] - } - } -} diff --git a/djapi/.serena/.gitignore b/djapi/.serena/.gitignore deleted file mode 100644 index 2e510af..0000000 --- a/djapi/.serena/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/cache -/project.local.yml diff --git a/djapi/.serena/memories/code_style_and_conventions.md b/djapi/.serena/memories/code_style_and_conventions.md deleted file mode 100644 index 4a363fe..0000000 --- a/djapi/.serena/memories/code_style_and_conventions.md +++ /dev/null @@ -1,23 +0,0 @@ -# Code Style & Conventions - -## Python -- Django app: all business logic in `api/stock/`, not in views -- views.py is thin forwarding layer: extract params -> call function -> return Response -- Double import pattern for standalone scripts: try relative import first, fall back to absolute -- Use `viewFunc_tsCodeAndDate()` wrapper for ts_code + date_range endpoints -- Use `viewFunc_singleParam()` wrapper for single-param endpoints -- DRF `@api_view(['GET'])` + `@extend_schema` on all views -- DRF `Response` (not `JsonResponse`) — no `safe=False` parameter -- Configuration split: config.py (token) / strategy_config.py / scan_config.py - -## Constraints -- `api/video/` is protected — do NOT modify unless user explicitly asks -- No python-dotenv dependency — use stdlib env loaders only -- Backward compatibility: keep re-exports when splitting modules -- Server `.env` file manages all secrets; uwsgi.ini only has DJANGO_SETTINGS_MODULE - -## Secrets -- All API keys/tokens/passwords via os.getenv() -- Local: .env file (not committed) -- Server: /home/simon/myquant/djapi/.env -- Django loads via djapi/env_loader.py, video loads via api/video/env.py diff --git a/djapi/.serena/memories/project_overview.md b/djapi/.serena/memories/project_overview.md deleted file mode 100644 index dd116ba..0000000 --- a/djapi/.serena/memories/project_overview.md +++ /dev/null @@ -1,32 +0,0 @@ -# Project Overview - -djapi is a Django 5.2 project providing financial data APIs for A-share stocks and CCTV news broadcast video processing. - -## Tech Stack -- Python 3.10, Django 5.2, uWSGI, nginx -- Tushare (stock data), akshare (alternative stock data) -- DRF + drf-spectacular (API documentation) -- MySQL (business data), SQLite (Django admin only) -- yt-dlp + ffmpeg + pydub (video/audio processing) -- DashScope (ASR), DeepSeek API (AI text processing) - -## Architecture -- Single Django app: `api` -- `api/stock/` — stock data module (Tushare/akshare -> pandas -> JsonResponse/DRF Response) -- `api/video/` — independent video processing pipeline (download -> audio -> ASR -> AI split -> MySQL) -- views.py is thin: extracts params, calls stock functions, returns Response - -## Key Files -- `api/views.py` — all ~15 API views, using @api_view + @extend_schema -- `api/stock/stock_utils.py` — shared utilities: tscodeCheck, viewFunc_tsCodeAndDate, viewFunc_singleParam -- `api/stock/config.py` — Tushare token + re-exports from strategy_config, scan_config -- `api/serializers.py` — 13 DRF Serializer classes -- `djapi/env_loader.py` — .env file loader (stdlib, no python-dotenv) -- `api/video/env.py` — standalone .env loader for video module -- `api/utils/mysql_handler.py` — shared MySQLDB class - -## Deployment -- Server: simon@doorcome.cn, path: /home/simon/myquant/djapi/ -- Virtual env: /opt/miniconda/envs/django/ -- uWSGI on port 5004, nginx reverse proxy -- Domains: api.doorcome.cn, echart.doorcome.cn diff --git a/djapi/.serena/memories/suggested_commands.md b/djapi/.serena/memories/suggested_commands.md deleted file mode 100644 index c5fabaf..0000000 --- a/djapi/.serena/memories/suggested_commands.md +++ /dev/null @@ -1,44 +0,0 @@ -# Suggested Commands - -## Development -```bash -python manage.py runserver 0.0.0.0:8000 # dev server -python manage.py check --deploy # check config -python manage.py test api # run tests -``` - -## uWSGI -```bash -uwsgi --ini uwsgi.ini # start -uwsgi --reload uwsgi.pid # hot reload -uwsgi --stop uwsgi.pid # stop -# On server: -/opt/miniconda/envs/django/bin/uwsgi --ini /home/simon/myquant/djapi/uwsgi.ini -kill $(lsof -ti:5004) # force stop -``` - -## Deploy -```bash -# Full sync (exclude production data) -rsync -avz --delete \ - --exclude='.env' --exclude='db.sqlite3' \ - --exclude='*.log' --exclude='uwsgi.pid' \ - --exclude='__pycache__/' --exclude='*.pyc' \ - --exclude='xwlb_video/' --exclude='audio_processing/' \ - /Users/summer/Downloads/cc-cursor/djapi/ \ - simon@doorcome.cn:/home/simon/myquant/djapi/ - -# Single file sync MUST use full target path -rsync -avz api/views.py simon@doorcome.cn:/home/simon/myquant/djapi/api/views.py -``` - -## API Docs -- /api/docs/ — Swagger UI -- /api/redoc/ — ReDoc -- /api/schema/ — OpenAPI JSON - -## Video Processing -```bash -cd api/video -python main.py -``` diff --git a/djapi/.serena/project.yml b/djapi/.serena/project.yml deleted file mode 100644 index 653c9f4..0000000 --- a/djapi/.serena/project.yml +++ /dev/null @@ -1,120 +0,0 @@ -# the name by which the project can be referenced within Serena -project_name: "djapi" - - -# list of languages for which language servers are started; choose from: -# al ansible bash clojure cpp -# cpp_ccls crystal csharp csharp_omnisharp dart -# elixir elm erlang fortran fsharp -# go groovy haskell haxe hlsl -# java json julia kotlin lean4 -# lua luau markdown matlab msl -# nix ocaml pascal perl php -# php_phpactor powershell python python_jedi python_ty -# r rego ruby ruby_solargraph rust -# scala solidity swift systemverilog terraform -# toml typescript typescript_vts vue yaml -# zig -# (This list may be outdated. For the current list, see values of Language enum here: -# https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py -# For some languages, there are alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.) -# Note: -# - For C, use cpp -# - For JavaScript, use typescript -# - For Free Pascal/Lazarus, use pascal -# Special requirements: -# Some languages require additional setup/installations. -# See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers -# When using multiple languages, the first language server that supports a given file will be used for that file. -# The first language is the default language and the respective language server will be used as a fallback. -# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored. -languages: -- typescript -- python - -# the encoding used by text files in the project -# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings -encoding: "utf-8" - -# line ending convention to use when writing source files. -# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default) -# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings. -line_ending: - -# The language backend to use for this project. -# If not set, the global setting from serena_config.yml is used. -# Valid values: LSP, JetBrains -# Note: the backend is fixed at startup. If a project with a different backend -# is activated post-init, an error will be returned. -language_backend: - -# whether to use project's .gitignore files to ignore files -ignore_all_files_in_gitignore: true - -# advanced configuration option allowing to configure language server-specific options. -# Maps the language key to the options. -# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. -# No documentation on options means no options are available. -ls_specific_settings: {} - -# list of additional paths to ignore in this project. -# Same syntax as gitignore, so you can use * and **. -# Note: global ignored_paths from serena_config.yml are also applied additively. -ignored_paths: [] - -# whether the project is in read-only mode -# If set to true, all editing tools will be disabled and attempts to use them will result in an error -# Added on 2025-04-18 -read_only: false - -# list of tool names to exclude. -# This extends the existing exclusions (e.g. from the global configuration) -# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html -excluded_tools: [] - -# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default). -# This extends the existing inclusions (e.g. from the global configuration). -# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html -included_optional_tools: [] - -# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools. -# This cannot be combined with non-empty excluded_tools or included_optional_tools. -# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html -fixed_tools: [] - -# list of mode names that are to be activated by default, overriding the setting in the global configuration. -# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. -# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply. -# Otherwise, this overrides the setting from the global configuration (serena_config.yml). -# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply -# for this project. -# This setting can, in turn, be overridden by CLI parameters (--mode). -# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes -default_modes: - -# list of mode names to be activated additionally for this project, e.g. ["query-projects"] -# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes. -# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes -added_modes: - -# initial prompt for the project. It will always be given to the LLM upon activating the project -# (contrary to the memories, which are loaded on demand). -initial_prompt: "" - -# time budget (seconds) per tool call for the retrieval of additional symbol information -# such as docstrings or parameter information. -# This overrides the corresponding setting in the global configuration; see the documentation there. -# If null or missing, use the setting from the global configuration. -symbol_info_budget: - -# list of regex patterns which, when matched, mark a memory entry as read‑only. -# Extends the list from the global configuration, merging the two lists. -read_only_memory_patterns: [] - -# list of regex patterns for memories to completely ignore. -# Matching memories will not appear in list_memories or activate_project output -# and cannot be accessed via read_memory or write_memory. -# To access ignored memory files, use the read_file tool on the raw file path. -# Extends the list from the global configuration, merging the two lists. -# Example: ["_archive/.*", "_episodes/.*"] -ignored_memory_patterns: [] diff --git a/djapi/CLAUDE.md b/djapi/CLAUDE.md index e8136a8..1c8b0e9 100644 --- a/djapi/CLAUDE.md +++ b/djapi/CLAUDE.md @@ -99,7 +99,7 @@ python api/video/main.py ## 注意事项 -- `config.py` 中的 TS_TOKEN 和 `deepseek.py`/`ai.py`/`audioRead.py` 中的 API key、`mysqlHandle.py` 中的数据库密码均为硬编码 —— 生产环境应迁移到环境变量 +- 所有密钥已迁移到环境变量,通过 `.env` 统一管理 - `api/stock/` 下的模块支持两种导入方式(相对导入和绝对导入),这是为了兼容「作为 Django app 被调用」和「直接命令行运行脚本」两种场景 - `api/video/` 模块设计为独立命令行运行,不依赖 Django 框架 - `db.sqlite3` 已提交到代码库,包含 Django admin 的用户数据 diff --git a/djapi/README.md b/djapi/README.md index f224c6d..f5342cc 100644 --- a/djapi/README.md +++ b/djapi/README.md @@ -98,7 +98,7 @@ uwsgi --stop uwsgi.pid | `news/reports/` | report_type, start_date, end_date, id | 日报查询(默认最近 24h;传 id 返回单份详情含事件) | | `news/events/` | days, importance, report_type, section, limit | 重要事件聚合(跨日报,最近 N 天 importance≥阈值) | -日报查询接口详细说明见 [`docs/news_report_api.md`](../docs/news_report_api.md)(表结构见 `djapi/docs/db_schema.md`)。 +日报查询接口详细说明见 [`docs/news_report_api.md`](../docs/news_report_api.md)(表结构见 `../docs/db_schema_v1.1.md`)。 API 文档(Swagger):`/api/docs/` OpenAPI Schema:`/api/schema/` diff --git a/djapi/api/report/query.py b/djapi/api/report/query.py index a15f567..581536c 100644 --- a/djapi/api/report/query.py +++ b/djapi/api/report/query.py @@ -1,7 +1,7 @@ """ -news_report / news_event 只读查询层(日报结构化入库,见 docs/db_schema.md)。 +news_report / news_event 只读查询层(日报结构化入库,见 docs/db_schema_v1.1.md)。 -连接配置来自环境变量(与 docs/report_db_design.md §7 保持一致): +连接配置来自环境变量(与 docs/db_schema_v1.1.md 保持一致): NEWS_DB_HOST / NEWS_DB_PORT / NEWS_DB_USER / NEWS_DB_PASSWORD / NEWS_DB_NAME NEWS_DB_PASSWORD 缺失时直接报错,禁止默认密码。 @@ -45,6 +45,18 @@ def _connect(): return mysql.connector.connect(**load_db_config()) +def _parse_json(value): + """把 JSON 字符串列(如 sources)解析为 dict/list;已是对象则原样返回。""" + if value is None: + return None + if isinstance(value, str): + try: + return json.loads(value) + except (TypeError, ValueError): + return None + return value + + def _row_to_dict(row: dict) -> dict: """序列化行:stats JSON 解析、日期/时间转 ISO 字符串。""" d = dict(row) @@ -87,11 +99,14 @@ def fetch_reports( report = _row_to_dict(row) cur.execute( "SELECT id, section, rank, importance, event_type, title, " - "summary, sentiment, source, url " + "summary, sentiment, source, sources, url " "FROM news_event WHERE report_id = %s ORDER BY section, rank", (report_id,), ) - report["events"] = [dict(r) for r in cur.fetchall()] + report["events"] = [ + {**dict(r), "sources": _parse_json(r["sources"])} + for r in cur.fetchall() + ] return report where, params = [], [] @@ -153,7 +168,7 @@ def fetch_important_events( sql = ( "SELECT r.report_date, r.report_type, e.id, e.section, e.rank, " "e.importance, e.event_type, e.title, e.summary, e.sentiment, " - "e.source, e.url " + "e.source, e.sources, e.url " "FROM news_event e " "JOIN news_report r ON r.id = e.report_id " "WHERE " + " AND ".join(where) @@ -163,6 +178,10 @@ def fetch_important_events( ) params.append(int(limit)) cur.execute(sql, tuple(params)) - return [dict(r) for r in cur.fetchall()] + rows = cur.fetchall() + return [ + {**dict(r), "sources": _parse_json(r["sources"])} + for r in rows + ] finally: conn.close() diff --git a/djapi/api/report/serializers.py b/djapi/api/report/serializers.py index 6c9f4db..13162a1 100644 --- a/djapi/api/report/serializers.py +++ b/djapi/api/report/serializers.py @@ -14,6 +14,7 @@ class EventSerializer(serializers.Serializer): summary = serializers.CharField(allow_null=True) sentiment = serializers.CharField(allow_null=True) source = serializers.CharField(allow_null=True) + sources = serializers.JSONField(allow_null=True) url = serializers.CharField(allow_null=True) @@ -47,4 +48,5 @@ class ImportantEventSerializer(serializers.Serializer): summary = serializers.CharField(allow_null=True) sentiment = serializers.CharField(allow_null=True) source = serializers.CharField(allow_null=True) + sources = serializers.JSONField(allow_null=True) url = serializers.CharField(allow_null=True) diff --git a/djapi/api/report/tests.py b/djapi/api/report/tests.py index 74b8a9c..17c018e 100644 --- a/djapi/api/report/tests.py +++ b/djapi/api/report/tests.py @@ -122,6 +122,17 @@ class NewsEventsAPITest(TestCase): self.assertEqual(kwargs['report_type'], 'intl') self.assertEqual(kwargs['section'], 'intl') + @patch('api.report.query.fetch_important_events', + return_value=[{'id': 1, 'title': 'x', 'source': 'yicai', + 'sources': ['yicai', 'stcn']}]) + def test_sources_field_passthrough(self, mock_fetch): + """事件聚合响应原样透传 sources(JSON 数组)""" + resp = self.client.get(self.url) + self.assertEqual(resp.status_code, 200) + body = resp.json() + self.assertEqual(body[0]['sources'], ['yicai', 'stcn']) + self.assertEqual(body[0]['source'], 'yicai') + def test_invalid_days(self): resp = self.client.get(self.url, {'days': 'abc'}) self.assertEqual(resp.status_code, 400) diff --git a/djapi/continuation.md b/djapi/continuation.md deleted file mode 100644 index 08030f4..0000000 --- a/djapi/continuation.md +++ /dev/null @@ -1,135 +0,0 @@ -# continuation.md - -## 当前项目状态 - -djapi — Django 5.2 金融数据 API 项目,2026-06-17 已部署。 - -## Checkpoint 记录 - -| 日期 | 内容 | -|------|------| -| 2026-08-03 | 新增日报查询 API ×2(news/reports/ + news/events/),基于 news_report/news_event 表,已部署 doorcome ✅ | - -服务器:`simon@doorcome.cn`,路径 `/home/simon/myquant/djapi/`,虚拟环境 `/opt/miniconda/envs/django/`。 - -## 已完成 - -### 1. 安全:密钥统一管理 -- 所有密钥 → 环境变量,`.env` 统一管理 -- `djapi/env_loader.py`(Django 端)+ `api/video/env.py`(video 端)双加载器 -- 共享 `MySQLDB` → `api/utils/mysql_handler.py` -- `.env.example`、`.gitignore` - -### 2. 代码质量 -- `api/views.py`:227 → ~130 行,消除重复 -- `api/stock/stock_utils.py`:`viewFunc_singleParam()` 包装器 -- `api/stock/config.py`:拆分为 config / strategy_config / scan_config - -### 3. drf-spectacular 集成 -- 14 端点 `@api_view` + `@extend_schema`,8 tag 分组 -- 13 Serializer,Swagger `/api/docs/` - -### 4. 股息率 API 优化 -- **删除** `api/stock/getDivData_AK.py`(akshare 版),`/api/getdivak/` 路由移除 -- **优化** `api/stock/getStockDiv2.py`: - - TTM 计算:`calculate_ttm_div` 行级循环 O(n²) → `rolling('360D').sum()` O(n) - - 删除向前填充逻辑(~30 行),避免与毛刺平滑冲突 -- **修复** `api/stock/smoothBrush.py`:if/elif 分支中 prev_valid/next_valid 赋值反了 - -### 5. video 模块重构与 Bug 修复 -- 新增 `api/video/env.py` — .env 加载 -- `newsRedo.py` 重写 — 三分支智能重处理 -- **P0 修复**:`getVideo5.py` 日期校验 bug(`start_date > start_date` → `start_date > end_date`) -- **P1 清理**:删除 `ai.py`(两个函数均为死代码),清理 `newsProcess.py` 冗余 import -- **P2 修复**:`audioRead.py` — `transcribe_audio` 异常时返回 `['', '']` 统一类型;`analyze_and_correct_text` 防御 None -- **P3 修复**:`newsProcess.py` — `news_to_db()` JSON 解析自适应 dict/list(DeepSeek json_object 模式返回 dict 包装) - -### 6. 文档与测试 -- `CLAUDE.md`、`README.md`、`continuation.md` -- 18 个单元测试 - -## video 目录文件现状(10 个 .py) - -| 文件 | 职责 | -|------|------| -| `env.py` | .env 加载 | -| `getVideo5.py` | 主流程:抓取→下载→ASR→入库 | -| `audioRead.py` | 音频转换、分割、ASR 识别、文本纠错 | -| `deepseek.py` | DeepSeek API 封装(类 + `deepseek_text` 函数,支持 `response_format`) | -| `newsProcess.py` | AI 新闻分割+标题提取,JSON 自适应解析 | -| `newsRedo.py` | 手动重处理(三分支) | -| `main.py` | 定时任务入口(当天) | -| `main_videos.py` | 批量补缺(扫描缺失日期) | -| `mysqlHandle.py` | MySQLDB 重新导出 | - -## 所有 API 端点(16 个) - -| 端点 | 数据源 | 说明 | -|------|--------|------| -| `stockbasic/` | Tushare | 日线行情 | -| `stockinfo/` | Tushare | 个股基本信息 | -| `industrys/` | Tushare | 行业股票列表 | -| `stockparam/` | Tushare | 个股参数 | -| `stockep/` | Tushare | TTM EPS | -| `quarterlyEps/` | Tushare | 季度 EPS | -| `indexByName/` | Tushare | 指数查询 | -| `indexDatas/` | Tushare | 指数行情 | -| `dailymargin/` | Tushare | 每日融资融券汇总 | -| `stockmargin/` | Tushare | 个股融资融券 | -| `finance/` | Tushare | 财务报表分析 | -| `getdiv/` | Tushare | 股息率(TTM rolling + 毛刺平滑) | -| `xwlbNews/` | MySQL | 新闻联播原始文本 | -| `xwlbFine/` | MySQL | 新闻联播 AI 精编 | -| `news/reports/` | MySQL (news_) | 日报查询:默认最近 24h;传 id 返回详情含事件 | -| `news/events/` | MySQL (news_) | 重要事件聚合:最近 N 天 importance≥阈值 | - ---- - -## 日报查询 API(2026-08-03 新增) - -### 模块 - -- 新增 `api/report/` 包(独立于 stock):`query.py`(连库+查询 SQL)/ `views.py`(2 视图)/ `serializers.py`(OpenAPI)/ `tests.py`(17 个单测,mock 查询层) -- `api/urls.py` 注册 `news/reports/`、`news/events/`;`settings.py` SPECTACULAR TAGS 加「日报」 -- 数据库:doorcome 本机 MariaDB `myquant` 库 `news_report`(180 行)+ `news_event`(4372 条),与现有 `MYSQL_*` 同库同用户 -- 连接配置:服务器 `djapi/.env` 新增 `NEWS_DB_*`(复用 MYSQL_* 值,密码必填否则 500) -- 文档:`docs/news_report_api.md`(使用手册,含线上地址/curl/真实样例);README API 概览表已加两行 - -### 部署(2026-08-03 完成) - -- rsync 增量同步(**未用文档中的 --delete**,见下)→ 重启 uWSGI → 冒烟通过(列表/详情/聚合/400/404) -- 线上:`https://api.doorcome.cn/api/news/reports/`、`/api/news/events/`,Swagger `/api/docs/`「日报」tag - -### 已知事项 - -1. **服务器顶层历史平铺文件未清理**:`/home/simon/myquant/djapi/` 顶层有 views.py/urls.py/smoothBrush.py/getStockDiv2.py/env_loader.py/akshare_data.py(历史 rsync 陷阱产物),`--delete` 会删除它们,但 `divSearch.py`(离线脚本)仍绝对导入顶层 getStockDiv2/smoothBrush → 本次增量同步保留;清理前需先修 divSearch.py 的导入 -2. **既有失败测试**:`api.tests.DateFormatCorrectionTest.test_empty_string`(date_format_correction('') 期望 None 实得 ''),与本次无关 -3. 冒烟曾发现 fetch_reports 列表 SQL 缺 `r.` 别名前缀(1052 ambiguous),已修复 -4. 本地验证需绕过 macOS TCC:`HOME=/tmp/djtest_home PYTHONPATH=/tmp/djtest_pkgs`(tushare 写 ~/tk.csv 被拦 + quant 环境缺 mysql-connector-python) - -## 部署 - -```bash -# 全量同步 -rsync -avz --delete \ - --exclude='.env' --exclude='db.sqlite3' \ - --exclude='*.log' --exclude='uwsgi.pid' \ - --exclude='__pycache__/' --exclude='*.pyc' \ - --exclude='xwlb_video/' --exclude='audio_processing/' \ - /Users/summer/Downloads/cc-cursor/djapi/ \ - simon@doorcome.cn:/home/simon/myquant/djapi/ - -# 单文件同步必须写完整路径 -# 正确:rsync api/views.py simon@...:/.../djapi/api/views.py - -# 重启 -ssh simon@doorcome.cn "kill \$(lsof -ti:5004); sleep 2; /opt/miniconda/envs/django/bin/uwsgi --ini /home/simon/myquant/djapi/uwsgi.ini" -``` - -## 关键设计决策 - -- video 模块保护、向后兼容优先、不使用 python-dotenv -- rsync 陷阱:多文件源会展平路径 -- `.env` 双加载:Django 端 `djapi/env_loader.py` + video 端 `api/video/env.py` -- 股息率 TTM 用 `rolling('360D').sum()` 向量化,不手动循环 -- DeepSeek json_object 模式返回 dict,newsProcess 自适应提取 list diff --git a/djapi/docs/report_db_design.md b/djapi/docs/report_db_design.md deleted file mode 100644 index 3626b34..0000000 --- a/djapi/docs/report_db_design.md +++ /dev/null @@ -1,330 +0,0 @@ -# Milestone 10 后端实现逻辑:日报结构化入库 - -> 版本:v0.1(设计稿) | 2026-07 -> 对应 project_plan.md「十八、Milestone 10」 -> **范围**:本项目侧"后端"= 数据生产层(日报内容生成 + 结构化写入 MySQL)。 -> 不包含 API 服务与前端页面(由用户另行实现),但表结构与数据契约以本文档为准,供 API/前端对接。 - ---- - -## 1. 定位 - -现有链路:`reporter.py` 收集数据 → `_render_html()` 渲染 HTML → scp 上传 doorcome。 -改造后:`reporter.py` 收集数据 → 组装结构化 `ReportData` → 写入 MySQL(`news_report` / `news_event`),不再产出 HTML。 - -另需:把 doorcome 上 178 份历史日报 HTML(`finance_news_daily_*` ×50、`intl_news_daily_*` ×128)解析成同一 `ReportData` 结构入库。 - ---- - -## 2. 数据流总览 - -``` -[历史 HTML ×178] [每日 pipeline] - doorcome:/var/www/html/echart/research/ crawler→extractor→dedup→llm→embed→qdrant - (一次性 scp 到 data/reports_history/) │ - │ ▼ - ▼ reporter.generate_report() - report_import/parser.py │ - │ (BeautifulSoup 解析) ▼ - ▼ 组装 ReportData 组装 ReportData - report_import/importer.py │ - │ (幂等 upsert) ▼ - ▼ │ - ┌────────────────────── MySQL (myquant 库) ──────────────────────┐ - │ news_report(主表) news_event(事件明细) │ - └────────────────────────────────────────────────────────────────┘ - ▲ - API / 前端(用户另行实现,只读) -``` - ---- - -## 3. 数据模型(Pydantic,`report_db/models.py`) - -```python -class EventRow(BaseModel): - """一条事件记录,对应 news_event 一行。""" - section: str # xwlb | news | cninfo | intl - rank: int # 板块内序号(从 1 开始) - importance: int | None = None - event_type: str | None = None - title: str - summary: str | None = None - sentiment: str | None = None # positive | negative | neutral | '' - source: str | None = None # 来源(如 cls / ForexLive) - url: str | None = None - -class ReportData(BaseModel): - """一份完整日报,对应 news_report 一行 + news_event 多行。""" - report_date: date # 日报日期(YYYY-MM-DD) - report_type: str # finance | intl - file_name: str # 源文件名(新生成时可为 "") - generated_at: datetime # 生成时间 - ai_summary: str | None = None - stats: dict[str, Any] = Field(default_factory=dict) # 数据总览统计快照 → JSON 列 - events: list[EventRow] = Field(default_factory=list) -``` - ---- - -## 4. 字段映射(核心契约) - -### 4.1 事件 JSON(data/events/)→ news_event - -现有事件文件结构与 news_event 字段对应关系(reporter 收集时直接转换): - -| news_event 字段 | 事件 JSON 来源 | -| --- | --- | -| section | 来源判定:`source_id=="cninfo"` → `cninfo`;`source_id=="xwlb"` → `xwlb`;否则 `news`;intl 解析固定 `intl` | -| importance | `event.importance` | -| event_type | `event.event_type` | -| title | `title` | -| summary | `event.summary` | -| sentiment | `event.sentiment` | -| source | `source_id` | -| url | `url`(xwlb 为空) | - -### 4.2 历史 HTML → ReportData - -解析策略:**表头驱动列映射**。不同日报表格列集合不同: - -| 板块 | 表格列(
| 包 | -版本 | -用途 | -
|---|---|---|
| pandas | -3.0 | -数据处理 | -
| numpy | -2.4 | -数值计算 | -
| akshare | -1.18 | -A 股数据获取 | -
| vectorbt | -1.0 | -回测引擎 | -
| optuna | -4.9 | -参数优化 | -
| lightgbm | -4.6 | -梯度提升模型 | -
| catboost | -1.2 | -梯度提升模型 | -
| scikit-learn | -1.9 | -特征工程 | -
| sqlalchemy | -2.0 | -数据库 ORM | -
| pymysql | -1.2 | -MySQL 连接 | -
所有代码从 finance/ 目录运行。Python 脚本开头加入:
import sys
-sys.path.insert(0, "/path/to/cc-cursor/finance")
-系统通过 SSH 隧道连接远程 MariaDB:
-bash shared/script/autossh.sh
-验证隧道:
-lsof -i :13306 | grep LISTEN
-# → ssh ... localhost:13306 (LISTEN) ...
-Host: 127.0.0.1
-Port: 13306
-User: myquant
-Password: <your-db-password>
-Database: myquant
-所有表使用 mac_ 前缀,与已有表隔离:
| 表名 | -内容 | -说明 | -
|---|---|---|
mac_stock_basic |
-A 股列表 | -5,524 只股票 | -
mac_stock_daily |
-日线行情 | -按需同步 | -
mac_stock_financial |
-财务指标 | -同花顺核心指标 | -
from database.connection import test_connection
-
-if test_connection():
- print("数据库连接成功")
-else:
- print("请先建立 SSH 隧道: bash shared/script/autossh.sh")
-from data.data_manager import DataManager
-
-dm = DataManager()
-dm.init_db() # 首次使用创建表(幂等操作)
-# 从 DB 缓存读取(已缓存的 5,524 只 A 股)
-stocks = dm.get_stock_list()
-# → DataFrame: index=ts_code, columns=[name, area, industry, ...]
-
-# 强制从 AkShare 刷新
-stocks = dm.get_stock_list(force_refresh=True)
-# 获取单只股票日线(DB 缓存优先,缺失自动补拉)
-daily = dm.get_daily("000001.SZ")
-# → DataFrame: trade_date, open, high, low, close, vol, amount, ...
-
-# 指定日期范围
-daily = dm.get_daily("000001.SZ", start="20240101", end="20241231")
-
-# 直接用索引
-price = daily.set_index("trade_date").sort_index()
-close = price["close"]
-fina = dm.get_financial("000001.SZ")
-# → DataFrame: end_date, eps, bvps, roe, net_profit_margin, debt_to_assets, ...
-# 数据源: stock_financial_abstract_ths(同花顺)
-# 覆盖: 主板/创业板/科创板
-# 增量同步:从 DB 最新日期到今天的缺失数据
-n = dm.sync_daily("000001.SZ")
-
-# 批量同步全部股票(谨慎使用,耗时长)
-total = dm.sync_all_daily()
-from factors.registry import get_factor, list_factors, list_categories
-
-# 查看所有因子分类
-print(list_categories())
-# → ['动量', 'RSI', 'MACD', '量价', '布林', 'ATR', '均线', '波动率', '换手率', '振幅', '基本面', '情绪']
-
-# 查看某个分类下的因子
-print(list_factors("RSI"))
-# → ['rsi_7', 'rsi_14']
-
-# 查看全部因子
-all_factors = list_factors()
-print(len(all_factors))
-# → 34
-# 按名称获取(使用默认参数)
-factor = get_factor("momentum_20") # 20 日动量
-factor = get_factor("rsi_14") # 14 日 RSI
-factor = get_factor("roe") # ROE 基本面因子
-factor = get_factor("news_sent_5") # 5 日新闻情绪因子
-
-# 自定义参数
-from factors.technical.momentum import MomentumFactor
-factor = MomentumFactor(period=60)
-from factors.engine import FactorEngine
-
-engine_fe = FactorEngine(dm)
-
-# 单股票多因子
-factors = [
- get_factor("momentum_20"),
- get_factor("rsi_14"),
- get_factor("volatility_20"),
- get_factor("ma_dev_20"),
-]
-factor_df = engine_fe.compute("000001.SZ", factors)
-# → DataFrame: index=trade_date, columns=[momentum_20, rsi_14, volatility_20, ma_dev_20]
-
-# 查看因子值
-print(factor_df.tail())
-print(factor_df.describe())
-# 计算多只股票在某一天的因子值
-cross = engine_fe.compute_universe(
- factors=[get_factor("momentum_20"), get_factor("rsi_14")],
- date="20250630",
- ts_codes=["000001.SZ", "600519.SH", "300750.SZ"],
-)
-# → DataFrame: index=ts_code, columns=[momentum_20, rsi_14]
-# 查看 NaN 率
-total = len(factor_df)
-for col in factor_df.columns:
- nan_pct = factor_df[col].isna().sum() / total * 100
- print(f"{col}: NaN {nan_pct:.1f}%")
-# 正常范围: 技术因子 0.3%-2.1%, 基本面因子 0%
-from backtest.vectorbt.engine import VectorBTEngine
-
-engine_bt = VectorBTEngine(
- initial_capital=100_000,
- commission=0.0003,
-)
-from backtest.strategies.rsi_mean_revert import RSIMeanRevertStrategy
-
-# 创建策略
-strategy = RSIMeanRevertStrategy(oversold=30, overbought=70)
-
-# 运行回测
-report = engine_bt.run(strategy, price_df, factor_df)
-# 一行摘要
-print(report.summary())
-# → 收益=29.4% 年化=4.3% 回撤=-19.1% 夏普=0.37 胜率=77.1% 交易=70笔
-
-# 字典格式
-metrics = report.to_dict()
-# → {'total_return': 29.4, 'cagr': 4.3, 'sharpe_ratio': 0.37, ...}
-
-# 获取净值曲线
-equity = report.equity_curve # pd.Series
-drawdown = report.drawdown_curve # pd.Series
-
-# 逐笔交易
-trades = report.trades_df # pd.DataFrame
-| 策略 | -类名 | -适用场景 | -
|---|---|---|
| 均线交叉 | -SMACrossStrategy(fast=5, slow=20) |
-趋势跟踪 | -
| RSI 反转 | -RSIMeanRevertStrategy(oversold=30, overbought=70) |
-均值回归 | -
| 动量突破 | -MomentumBreakoutStrategy(lookback=20, exit_period=10) |
-动量策略 | -
| 因子阈值 | -FactorCrossStrategy(factor_column, buy_threshold, sell_threshold) |
-通用因子 | -
| 因子轮动 | -FactorRotationStrategy(factor_name, top_n=5) |
-截面选股 | -
from backtest.base import BaseStrategy
-
-class MyStrategy(BaseStrategy):
- name = "my_strategy"
- category = "custom"
-
- def __init__(self, param_a=10):
- self.param_a = param_a
-
- def generate_signals(self, factor_df):
- # factor_df 包含因子值和 close 列
- # 返回: 1=买入, 0=卖出, -1=持有
- signals = pd.Series(-1, index=factor_df.index)
- signals[factor_df["rsi_14"] < 30] = 1 # RSI 超卖买入
- signals[factor_df["rsi_14"] > 70] = 0 # RSI 超买卖出
- return signals
-
-report = engine_bt.run(MyStrategy(param_a=20), price_df, factor_df)
-report_xs = engine_bt.run_cross_section(
- strategy,
- price_universe={"000001.SZ": df1, "600519.SH": df2},
- factor_universe={"000001.SZ": f1, "600519.SH": f2},
-)
-# → 等权组合回测报告
-from optimizer.engine import OptunaEngine
-from optimizer.space import rsi_revert_space, sma_cross_space
-from backtest.strategies.rsi_mean_revert import RSIMeanRevertStrategy
-
-opt_engine = OptunaEngine(engine_bt)
-
-# 优化 RSI 反转策略参数
-result = opt_engine.optimize(
- strategy_class=RSIMeanRevertStrategy,
- search_space=rsi_revert_space,
- price_df=price_df,
- factor_df=factor_df,
- metric="sharpe", # 优化目标: sharpe/cagr/calmar/total_return
- n_trials=200, # 试验次数
-)
-print(result.summary())
-# → 最优参数: oversold=13, overbought=66
-# → 最优目标 (sharpe): 0.5985
-
-# 最优参数的回测报告
-best_report = result.best_report
-
-# 参数重要性
-for k, v in sorted(result.param_importance.items(), key=lambda x: -x[1]):
- print(f" {k}: {v:.4f}")
-
-# 试验记录
-trials = result.trials_df # pd.DataFrame
-wf_result = opt_engine.optimize_walk_forward(
- strategy_class=RSIMeanRevertStrategy,
- search_space=rsi_revert_space,
- price_df=price_df,
- factor_df=factor_df,
- metric="sharpe",
- n_trials=80,
- train_window=252 * 3, # 3 年训练
- test_window=252, # 1 年测试
-)
-print(wf_result.summary())
-# → 各窗口参数变化 + 整体收益
-from optimizer.presets import (
- optimize_sma_cross,
- optimize_rsi_revert,
- optimize_momentum_breakout,
-)
-
-result = optimize_rsi_revert(price_df, factor_df, engine_bt, n_trials=100)
-from optimizer.space import SearchSpace
-
-my_space = SearchSpace(params=[
- {"name": "fast", "type": "int", "low": 2, "high": 30, "step": 1},
- {"name": "slow", "type": "int", "low": 15, "high": 120, "step": 5},
-])
-result = opt_engine.optimize(MyStrategy, my_space, price_df, factor_df)
-from models.features import FeatureEngine
-
-# lookahead=5: 预测未来 5 个交易日收益
-fe = FeatureEngine(lookahead=5, label_type="regression")
-
-# 构建特征矩阵和标签
-X, y = fe.build(factor_df, price_df, fit=True)
-# → X: 标准特征矩阵(去极值 → 缺失填充 → RobustScaler)
-# → y: 未来 5 日收益率(%)
-
-print(f"特征: {X.shape[1]} 列, 样本: {X.shape[0]} 行")
-print(f"标签: mean={y.mean():.2f}%, std={y.std():.2f}%")
-# 时间序列划分(前 70% 训练,后 30% 测试)
-n = len(X)
-split = int(n * 0.7)
-X_train, X_test = X.iloc[:split], X.iloc[split:]
-y_train, y_test = y.iloc[:split], y.iloc[split:]
-
-print(f"训练集: {len(X_train)} 行")
-print(f"测试集: {len(X_test)} 行")
-from models.lightgbm.model import LightGBMModel
-
-model = LightGBMModel(
- params={
- "n_estimators": 200,
- "learning_rate": 0.03,
- "num_leaves": 15,
- },
- early_stopping=100,
- eval_ratio=0.2, # 20% 做验证集
-)
-
-model.fit(X_train, y_train)
-pred = model.predict(X_test)
-
-# 评估
-ic = pred.corr(y_test)
-print(f"测试集 IC: {ic:.4f}")
-from models.catboost.model import CatBoostModel
-
-model = CatBoostModel(
- params={"iterations": 200, "learning_rate": 0.03, "depth": 5},
- eval_ratio=0.2,
-)
-model.fit(X_train, y_train)
-pred = model.predict(X_test)
-# LightGBM
-imp = model.get_feature_importance(importance_type="gain")
-print(imp.head(10))
-# → feature, importance, importance_pct
-
-# CatBoost
-imp = model.get_feature_importance()
-print(imp.head(5))
-# 5 折时间序列 CV(不 shuffle)
-cv_df = model.cv_evaluate(X_train, y_train, n_folds=5)
-print(cv_df)
-# → 各折 IC + MSE, 均值
-# 保存
-model.save("models/lightgbm_000001.pkl")
-
-# 加载
-model = LightGBMModel.load("models/lightgbm_000001.pkl")
-from models.backtest_integration import MLStrategy, MLBenchmark
-
-# 预测值分位 → 交易信号
-strategy = MLStrategy(
- model=model,
- feature_engine=fe,
- buy_quantile=0.7, # 预测值最高的 30% 买入
- sell_quantile=0.3, # 预测值最低的 30% 卖出
- rebalance_freq=5, # 每 5 日调仓
-)
-report = engine_bt.run(strategy, price_df, factor_df)
-
-# 多模型对比
-benchmark = MLBenchmark(
- models=[lgb_model, cb_model],
- feature_engine=fe,
- price_df=test_price,
- factor_df=test_factor,
-)
-df = benchmark.run()
-print(df)
-# → model × (IC, total_return, sharpe, win_rate, trades)
-编辑 finance/.env:
# DashScope API(推荐)
-QWEN_API_KEY=sk-your-key-here
-QWEN_MODEL=qwen-turbo
-
-# 或本地 Ollama
-# QWEN_LOCAL_BASE_URL=http://localhost:11434/v1
-# QWEN_LOCAL_MODEL=qwen2.5:7b
-# 按指数成分股分析(沪深300 + 中证500)
-SENTIMENT_SCOPE_TYPE=index
-SENTIMENT_SCOPE_INDEXES=000300,000905
-
-# 按板块分析
-# SENTIMENT_SCOPE_TYPE=sector
-# SENTIMENT_SCOPE_SECTORS=银行,电力设备,医药生物
-
-# 按自定义列表
-# SENTIMENT_SCOPE_TYPE=custom
-# SENTIMENT_SCOPE_CUSTOM=000001.SZ,600519.SH,300750.SZ
-from factors.sentiment.sentiment_engine import SentimentEngine
-from factors.sentiment.news_source import NewsSource
-from factors.sentiment.qwen_client import QwenClient
-
-sent = SentimentEngine(dm, qwen_client=QwenClient(), news_source=NewsSource())
-
-# 单股票情绪因子
-sent_df = sent.compute("000001.SZ", max_news=20)
-# → DataFrame: (trade_date, news_sent_5, news_conf_5, sent_delta_5)
-
-# 批量计算
-results = sent.compute_batch(
- ts_codes=["000001.SZ", "600519.SH", "300750.SZ"],
- max_news=10,
-)
-系统聚合三个数据源:
-| 数据源 | -说明 | -配置 | -
|---|---|---|
AkShare stock_news_em |
-东方财富个股新闻 | -use_akshare=True |
-
MariaDB xwlb_daily_ext |
-新闻联播分割数据 | -use_xwlb=True |
-
MCP trendradar-news |
-外部新闻聚合服务 | -use_mcp=True |
-
news = NewsSource(
- use_akshare=True, # 启用东方财富
- use_xwlb=True, # 启用新闻联播
- use_mcp=False, # 关闭 MCP
-)
-
-news_df = news.fetch("000001.SZ", start="20260501", end="20260603")
-# → DataFrame: date, title, content, source, url
-发布时间 直接保留 → align_news_to_trading_days 对齐到最近交易日news_date + 1 day(晚间播出 → 次日市场影响)→ 对齐到交易日周五新闻联播 → +1 = 周六 → align → 下周一交易日
-from agents.orchestrator import AgentOrchestrator
-
-engines = {
- "dm": dm,
- "fe": engine_fe,
- "bt": engine_bt,
- "opt": opt_engine,
- "sent": sent,
-}
-
-orch = AgentOrchestrator(**engines)
-orch.setup()
-# → [Orchestrator] 已注册 4 个 Agent: ['research', 'selection', 'risk', 'report']
-# 完整每日流程(同步行情 → 风险评估 → 选股打分 → 生成日报)
-python finance/cli/agent_cli.py daily
-
-# 今日选股 Top 15
-python finance/cli/agent_cli.py picks 15
-
-# 风险评估
-python finance/cli/agent_cli.py risk
-
-# 因子研究(IC 评估)
-python finance/cli/agent_cli.py research
-
-# 生成指定日期日报
-python finance/cli/agent_cli.py report 20260603
-============================================================
-[Orchestrator] 每日流程 — 20260603
-============================================================
-
-[Step 1/4] 同步行情... 0 条(已是最新)
-[Step 2/4] 风险评估... high, 仓位 30%
-[Step 3/4] 股票打分... 1 只
-[Step 4/4] 生成日报... reports/daily_20260603.md
-日报保存到 finance/reports/daily_YYYYMMDD.md,内容包含:
# 各 Agent 独立调用
-selection_result = orch.picks(date="20260603", top_n=15)
-risk_result = orch.risk_check()
-research_result = orch.run_research_cycle()
-report_result = orch.generate_report(date="20260603")
-finance/.env)# ── Qwen API ──────────────────────
-QWEN_API_KEY=sk-xxx # DashScope API Key
-QWEN_MODEL=qwen-turbo # 模型选择: qwen-turbo/plus/max
-
-# ── 本地 Ollama(可选) ──────────
-# QWEN_LOCAL_BASE_URL=http://localhost:11434/v1
-# QWEN_LOCAL_MODEL=qwen2.5:7b
-
-# ── 数据库 ───────────────────────
-MAC_DB_HOST=127.0.0.1
-MAC_DB_PORT=13306
-MAC_DB_USER=myquant
-MAC_DB_PASSWORD=<your-db-password>
-MAC_DB_NAME=myquant
-
-# ── 情绪分析范围 ─────────────────
-SENTIMENT_SCOPE_TYPE=index
-SENTIMENT_SCOPE_INDEXES=000300,000905
-SENTIMENT_MAX_NEWS_PER_STOCK=20
-
-# ── MCP 新闻服务(可选) ─────────
-NEWS_MCP_URL=http://192.168.1.160:3333/mcp
-VectorBTEngine(
- initial_capital=100_000, # 初始资金(元)
- commission=0.0003, # 手续费(万三)
-)
-opt_engine.optimize(
- n_trials=200, # 试验次数
- metric="sharpe", # 优化目标
- # 可选: cagr, calmar, total_return, return_over_dd, win_rate, profit_factor
-)
-# LightGBM 推荐参数
-LightGBMModel(params={
- "n_estimators": 200,
- "learning_rate": 0.03,
- "num_leaves": 15,
- "min_data_in_leaf": 20,
- "feature_fraction": 0.7,
- "bagging_fraction": 0.7,
-})
-
-# CatBoost 推荐参数
-CatBoostModel(params={
- "iterations": 200,
- "learning_rate": 0.03,
- "depth": 5,
- "min_data_in_leaf": 20,
-})
-import sys; sys.path.insert(0, "finance")
-
-from data.data_manager import DataManager
-from factors.registry import get_factor
-from factors.engine import FactorEngine
-from backtest.vectorbt.engine import VectorBTEngine
-from backtest.strategies.rsi_mean_revert import RSIMeanRevertStrategy
-
-# 数据
-dm = DataManager(); dm.init_db()
-price = dm.get_daily("000001.SZ").set_index("trade_date")
-
-# 因子
-engine_fe = FactorEngine(dm)
-factor_df = engine_fe.compute("000001.SZ", [get_factor("rsi_14")])
-
-# 回测
-engine_bt = VectorBTEngine()
-report = engine_bt.run(
- RSIMeanRevertStrategy(oversold=30, overbought=70),
- price, factor_df,
-)
-print(report.summary())
-from optimizer.engine import OptunaEngine
-from optimizer.space import rsi_revert_space
-
-opt_engine = OptunaEngine(engine_bt)
-
-# 寻优
-result = opt_engine.optimize(
- RSIMeanRevertStrategy, rsi_revert_space,
- price, factor_df, metric="sharpe", n_trials=200,
-)
-print(result.summary())
-
-# Walk-Forward 验证
-wf = opt_engine.optimize_walk_forward(
- RSIMeanRevertStrategy, rsi_revert_space,
- price, factor_df, n_trials=80,
- train_window=756, test_window=252,
-)
-print(wf.summary())
-from models.features import FeatureEngine
-from models.lightgbm.model import LightGBMModel
-from models.backtest_integration import MLStrategy
-
-# 特征工程
-fe = FeatureEngine(lookahead=5)
-X, y = fe.build(factor_df, price, fit=True)
-split = int(len(X) * 0.7)
-
-# 训练
-model = LightGBMModel(params={"n_estimators": 200, "learning_rate": 0.03})
-model.fit(X.iloc[:split], y.iloc[:split])
-
-# 回测
-strategy = MLStrategy(model, fe)
-report = engine_bt.run(strategy, price, factor_df)
-print(report.summary())
-print(model.get_feature_importance().head(5))
-from agents.orchestrator import AgentOrchestrator
-
-orch = AgentOrchestrator(
- dm=dm, fe=engine_fe, bt=engine_bt, opt=opt_engine, sent=sent,
-)
-orch.setup()
-results = orch.run_daily()
-
-# 获取结果
-sel = results["selection"]
-risk = results["risk"]
-report_path = results["report"]["report_path"]
-print(f"日报: {report_path}")
-# 检查端口
-lsof -i :13306 | grep LISTEN
-
-# 重新建立
-bash shared/script/autossh.sh
-这是 AkShare 的 curl_cffi 在连续请求时偶发的连接问题。系统已内置 3 次递增间隔重试 + fallback 机制,通常第 2-3 次重试会成功。如果持续失败:
-dm.get_financial(ts_code))QWEN_API_KEYOptunaEngine.optimize() 寻找更优参数factor_df.describe())当验证集损失不下降时,早停会在很少的迭代后触发。这是单股票预测的正常现象(信号噪声比低)。建议:
-eval_ratio=0.0 禁用早停learning_rate 到 0.01min_data_in_leaf 防止过拟合.env 中 QWEN_API_KEY 已配置dashscope.aliyuncs.comcurl http://localhost:11434/api/tags日报只对 DB 中有日线缓存的股票打分。需要先同步目标股票池的数据:
-# 同步单只
-dm.sync_daily("000001.SZ")
-
-# 按范围批量同步(需先配置 SENTIMENT_SCOPE)
-codes = sent.get_scope_stocks()
-for code in codes[:10]:
- dm.sync_daily(code)
-所有脚本位于 finance/cli/,需在项目根目录或 finance/ 下运行。
agent_cli.pycd finance && python cli/agent_cli.py <命令> [参数]
-| 命令 | -说明 | -示例 | -
|---|---|---|
daily [DATE] |
-完整每日流程(增量同步已缓存→评估风险→选股→日报) | -agent_cli.py daily |
-
picks [N] [DATE] |
-多因子选股 Top N(需已缓存) | -agent_cli.py picks 15 |
-
risk |
-市场风险评估(等级、仓位、止损) | -agent_cli.py risk |
-
research |
-因子发现:遍历因子计算 IC/IC_IR 排名 | -agent_cli.py research |
-
report [DATE] |
-生成日报(含三指数行情+选股+情绪+风险评估) | -agent_cli.py report |
-
warmup [N] |
-首次批量预热范围股票到 DB 缓存(每批 N 只,默认 50) | -agent_cli.py warmup 50 |
-
daily 流程:
[Step 1/4] 增量同步 → 只更新已缓存股票(最新则 0.04s 跳过)
- → 未缓存提示:运行 'agent_cli.py warmup' 首次预热
-[Step 2/4] 风险评估 → high/medium/low + 仓位建议 + 预警
-[Step 3/4] 股票打分 → DB 缓存命中率 + 多因子等权打分 → Top 15
-[Step 4/4] 日报生成 → 三指数行情 (Tushare) + 情绪摘要 + 风险预警
- → reports/daily_YYYYMMDD.md
-数据源优先级:Tushare → AkShare(.env 配置 TUSHARE_TOKEN)
demo_data_manager.pypython cli/demo_data_manager.py [--ts_code CODE] [--start YYYYMMDD]
-| 参数 | -默认值 | -说明 | -
|---|---|---|
--ts_code |
-000001.SZ |
-测试股票代码 | -
--start |
-20250101 |
-起始日期 YYYYMMDD | -
5 步验证:数据库连接 → 建表 → 股票列表 → 日线获取(双源fallback) → 增量同步。
-demo_factor_engine.pypython cli/demo_factor_engine.py [--ts_code CODE] [--ts_code2 CODE]
-| 参数 | -默认值 | -说明 | -
|---|---|---|
--ts_code |
-000001.SZ |
-测试股票代码 | -
--ts_code2 |
-600519.SH |
-截面测试第二只股票 | -
验证:因子注册表(12分类/34因子)→ 技术因子计算(describe统计) → 基本面因子(ROE/PE/PB/EP) → NaN 覆盖率检查 → 双股票截面因子。
-demo_backtest.pypython cli/demo_backtest.py [--ts_code CODE]
-| 参数 | -默认值 | -说明 | -
|---|---|---|
--ts_code |
-000001.SZ |
-回测股票代码 | -
测试 5 个内置策略:
-| 策略 | -参数 | -
|---|---|
| SMACrossStrategy | -(5,20) / (10,60) | -
| RSIMeanRevertStrategy | -(30,70) / (20,80) | -
| MomentumBreakoutStrategy | -lookback=20 | -
| FactorCrossStrategy | -momentum_20 > 0 | -
| FactorRotationStrategy | -momentum top 20% | -
demo_optimizer.pypython cli/demo_optimizer.py [--ts_code CODE] [--trials N]
-| 参数 | -默认值 | -说明 | -
|---|---|---|
--ts_code |
-000001.SZ |
-回测股票代码 | -
--trials |
-200 |
-Optuna 试验次数 | -
对 RSI 反转策略执行参数寻优 + Walk-Forward 验证。输出最优 vs 默认对比表 + 参数重要性排序。
-demo_ml.pypython cli/demo_ml.py [--ts_code CODE] [--lookahead N]
-| 参数 | -默认值 | -说明 | -
|---|---|---|
--ts_code |
-000001.SZ |
-训练股票代码 | -
--lookahead |
-5 |
-预测未来 N 日收益 | -
完整 ML pipeline:特征工程(25因子→Winsorize→RobustScaler) → LightGBM训练(IC/CV) → CatBoost训练 → MLBenchmark对比(IC/收益/夏普/胜率)。
-demo_sentiment.pypython cli/demo_sentiment.py [--ts_code CODE] [--no-qwen]
-| 参数 | -默认值 | -说明 | -
|---|---|---|
--ts_code |
-000001.SZ |
-测试股票代码 | -
--no-qwen |
-flag | -跳过 Qwen API 调用 | -
6 步验证:新闻数据源(三源聚合) → 日期对齐 → Qwen 客户端状态 → SentimentEngine全链路 → 分析范围解析。
-适合快速检查情绪因子系统是否就绪。
-demo_sentiment_detail.pypython cli/demo_sentiment_detail.py [选项]
-最详细的情绪因子脚本,支持完整命令行参数和逐步输出。
-| 参数 | -类型 | -默认值 | -说明 | -
|---|---|---|---|
--ts_code |
-str | -000001.SZ |
-股票代码,多个用逗号分隔 | -
--date |
-str | -今天 | -目标日期 YYYYMMDD | -
--start |
-str | -date-30天 | -起始日期 YYYYMMDD | -
--end |
-str | -date | -结束日期 YYYYMMDD | -
--scope-type |
-str | -- | -分析范围:index/sector/custom/all |
-
--scope-indexes |
-str | -000300 |
-指数代码(逗号分隔) | -
--scope-sectors |
-str | -- | -板块名称(逗号分隔) | -
--max-news |
-int | -.env 配置 | -最大新闻条数 | -
--max-analyze |
-int | -50 |
-Qwen API 分析最大条数(控制成本) | -
--no-xwlb |
-flag | -- | -禁用新闻联播数据源 | -
--no-akshare |
-flag | -- | -禁用东方财富数据源 | -
--no-mcp |
-flag | -- | -禁用 MCP 数据源 | -
--source |
-str | -- | -仅用指定数据源:xwlb/akshare/mcp |
-
--no-qwen |
-flag | -- | -跳过 Qwen API 调用(仅演示数据流) | -
使用示例:
-# 默认演示(000001.SZ,最近30天,全数据源)
-python cli/demo_sentiment_detail.py
-
-# 指定股票和日期
-python cli/demo_sentiment_detail.py --ts_code 600519.SH --date 20260603
-
-# 多股票 + 日期范围
-python cli/demo_sentiment_detail.py --ts_code 000001.SZ,300316.SZ --start 20260501 --end 20260603
-
-# 按指数成分股分析
-python cli/demo_sentiment_detail.py --scope-type index --scope-indexes 000300
-
-# 按板块分析
-python cli/demo_sentiment_detail.py --scope-type sector --scope-sectors 银行,电力设备
-
-# 只看东方财富新闻,不调用 Qwen
-python cli/demo_sentiment_detail.py --source akshare --no-qwen --max-news 20
-输出 6 步详情:
-Step 0: 初始化引擎(显示数据源、API状态、范围、Tushare可用性)
-Step 1: 按数据源分别拉取新闻(xwlb/AkShare/MCP 各自数量 + 双源fallback)
-Step 2: 新闻详情(按来源分开展示标题/内容/链接)
-Step 3: 日期对齐(xwlb +1day偏移 + 非交易日对齐 + DB缓存检查→sync补齐)
-Step 4: Qwen 情绪分析(每条新闻的分数/置信度/主题/来源标签)
-Step 5: 因子计算(weighted sent / confidence-weighted / momentum + 公式说明)
-Step 6: 结果输出(因子值表 + 历史统计 + 每新闻情绪贡献明细)
-| 脚本 | -参数 | -用途 | -数据源 fallback | -
|---|---|---|---|
agent_cli.py |
-子命令 + 参数 | -日常操作入口 | -✅ | -
demo_data_manager.py |
---ts_code --start |
-Sprint 0 验证 | -✅ | -
demo_factor_engine.py |
---ts_code --ts_code2 |
-Sprint 1 验证 | -✅ | -
demo_backtest.py |
---ts_code |
-Sprint 2 验证 | -✅ | -
demo_optimizer.py |
---ts_code --trials |
-Sprint 3 验证 | -✅ | -
demo_ml.py |
---ts_code --lookahead |
-Sprint 4 验证 | -✅ | -
demo_sentiment.py |
---ts_code --no-qwen |
-Sprint 5 快速验证 | -✅ | -
demo_sentiment_detail.py |
-14 个 argparse 参数 | -Sprint 5 详细演示 | -✅ | -