Initial commit: cc-cursor 全链路量化研究平台

7 Sprints 全部完成:
  Sprint 0: 基础设施 (DataManager + MariaDB)
  Sprint 1: 因子引擎 (34因子/12分类)
  Sprint 2: VectorBT 回测 (5策略+截面)
  Sprint 3: Optuna 优化 (+Walk-Forward)
  Sprint 4: ML 模型 (LightGBM+CatBoost)
  Sprint 5: Qwen 情绪因子 (三源新闻+日期对齐)
  Sprint 6: Agent 系统 (4Agent+日报.md/.html)

生产加固 (15项): Tushare双源fallback, SSH自动恢复, pool_pre_ping,
  save_daily先删后插, load_dotenv绝对路径, 日报5d/20d修复,
  RiskAgent改上证指数, 昨日对比+数据截止, mac_report utf8mb4,
  CLAUDE-*.md 9条已知Bug, demo全参数化, djapi数据源归一化,
  indexDatas API修正

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-06-07 15:59:05 +08:00
co-authored by Claude Opus 4.7
commit 271a9343a5
293 changed files with 59598 additions and 0 deletions
+453
View File
@@ -0,0 +1,453 @@
"""
情绪因子详细运行过程演示。
用法:
# 默认:000001.SZ,最近30天,指数范围
python cli/demo_sentiment_detail.py
# 指定股票代码和日期
python cli/demo_sentiment_detail.py --ts_code 600519.SH --date 20260603
python cli/demo_sentiment_detail.py --ts_code 000001.SZ,600519.SH,300750.SZ
# 指定日期范围
python cli/demo_sentiment_detail.py --start 20260501 --end 20260603
# 分析指定指数成分股
python cli/demo_sentiment_detail.py --scope-type index --scope-indexes 000300
# 分析指定板块
python cli/demo_sentiment_detail.py --scope-type sector --scope-sectors 银行,电力设备
# 只使用特定新闻源
python cli/demo_sentiment_detail.py --no-xwlb --no-mcp
python cli/demo_sentiment_detail.py --source akshare
# 跳过 Qwen API 调用(仅演示数据流)
python cli/demo_sentiment_detail.py --no-qwen
"""
import sys, os, json, argparse
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import pandas as pd
import numpy as np
from datetime import datetime
def parse_args():
p = argparse.ArgumentParser(description="情绪因子详细运行过程演示")
p.add_argument("--ts_code", default="000001.SZ",
help="股票代码,多个用逗号分隔(默认: 000001.SZ)")
p.add_argument("--date", default=None,
help="目标日期 YYYYMMDD(默认: 今天)")
p.add_argument("--start", default=None,
help="起始日期 YYYYMMDD(默认: date-30天)")
p.add_argument("--end", default=None,
help="结束日期 YYYYMMDD(默认: date 或今天)")
p.add_argument("--scope-type", default=None,
choices=["index", "sector", "custom", "all"],
help="分析范围类型(覆盖 ts_code")
p.add_argument("--scope-indexes", default="000300",
help="指数代码,逗号分隔(默认: 000300)")
p.add_argument("--scope-sectors", default="",
help="板块名称,逗号分隔")
p.add_argument("--max-news", type=int, default=None,
help="最大新闻条数(默认: .env SENTIMENT_MAX_NEWS_PER_STOCK 或 30")
p.add_argument("--max-analyze", type=int, default=50,
help="Qwen 分析最大条数(默认: 50,控制成本)")
p.add_argument("--no-xwlb", action="store_true", help="禁用新闻联播数据源")
p.add_argument("--no-akshare", action="store_true", help="禁用东方财富数据源")
p.add_argument("--no-mcp", action="store_true", help="禁用 MCP 数据源")
p.add_argument("--source", default=None,
choices=["xwlb", "akshare", "mcp"],
help="仅使用指定数据源")
p.add_argument("--no-qwen", action="store_true", help="跳过 Qwen 分析(仅演示数据流)")
return p.parse_args()
def main():
args = parse_args()
date = args.date or datetime.now().strftime("%Y%m%d")
start = args.start or (datetime.strptime(date, "%Y%m%d") - pd.Timedelta(days=30)).strftime("%Y%m%d")
end = args.end or date
print("=" * 72)
print(" 情绪因子详细运行过程")
print("=" * 72)
print(" 日期: {} ~ {} (目标: {})".format(start, end, date))
# ═══════════════════════════════════════════════════════════════
# Step 0: 初始化
# ═══════════════════════════════════════════════════════════════
print("\n" + "-" * 72)
print("Step 0: 初始化引擎")
print("-" * 72)
from data.data_manager import DataManager
from factors.sentiment.qwen_client import QwenClient
from factors.sentiment.news_source import NewsSource, align_news_to_trading_days
dm = DataManager()
dm.init_db()
client = QwenClient()
has_api = (bool(client.api_key) or bool(client.local_base_url)) and not args.no_qwen
use_xwlb = not args.no_xwlb and (args.source is None or args.source == "xwlb")
use_akshare = not args.no_akshare and (args.source is None or args.source == "akshare")
use_mcp = not args.no_mcp and (args.source is None or args.source == "mcp")
print(" Qwen API: {}".format("DashScope/{}".format(client.model) if (has_api and not client.local_base_url) else (
"本地 Ollama/{}".format(client.local_model) if (has_api and client.local_base_url) else "跳过(--no-qwen 或未配置)")))
print(" 数据源: {}/{}/{}".format(
"xwlb" if use_xwlb else "xwlb(off)",
"akshare" if use_akshare else "akshare(off)",
"mcp" if use_mcp else "mcp(off)",
))
max_news = args.max_news or int(os.getenv("SENTIMENT_MAX_NEWS_PER_STOCK", "30"))
print(" 最大新闻: {} 条 (SENTIMENT_MAX_NEWS_PER_STOCK={})".format(
max_news, os.getenv("SENTIMENT_MAX_NEWS_PER_STOCK", "未设置")))
# 分析范围
if args.scope_type:
from factors.sentiment.sentiment_engine import SentimentEngine
# 临时覆盖环境变量
os.environ["SENTIMENT_SCOPE_TYPE"] = args.scope_type
if args.scope_indexes:
os.environ["SENTIMENT_SCOPE_INDEXES"] = args.scope_indexes
if args.scope_sectors:
os.environ["SENTIMENT_SCOPE_SECTORS"] = args.scope_sectors
sent_tmp = SentimentEngine(dm)
ts_codes = sent_tmp.get_scope_stocks()
print(" 分析范围: {} ({})".format(args.scope_type, len(ts_codes)))
if len(ts_codes) > 10:
print(" 股票示例: {}... (共 {} 只)".format(", ".join(ts_codes[:10]), len(ts_codes)))
else:
print(" 股票: {}".format(", ".join(ts_codes)))
else:
ts_codes = [c.strip() for c in args.ts_code.split(",") if c.strip()]
# ═══════════════════════════════════════════════════════════════
# Step 1: 分别从三个数据源获取新闻
# ═══════════════════════════════════════════════════════════════
print("\n" + "-" * 72)
print("Step 1: 获取新闻 ({} 只股票)".format(len(ts_codes)))
print("-" * 72)
all_raw = []
for ts_code in ts_codes:
print("\n --- {} ---".format(ts_code))
xwlb_raw = pd.DataFrame()
ak_raw = pd.DataFrame()
mcp_raw = pd.DataFrame()
if use_xwlb:
try:
xwlb_src = NewsSource(use_akshare=False, use_mcp=False)
xwlb_raw = xwlb_src.fetch(ts_code, start=start, end=end, max_news=max_news * 3)
print(" 新闻联播(DB xwlb_daily_ext): {} 条 (news_date范围: {}-1~{}-1)".format(
len(xwlb_raw), start, end))
except Exception as e:
print(" 新闻联播: 获取失败 ({})".format(e))
if use_akshare:
try:
ak_src = NewsSource(use_xwlb=False, use_mcp=False)
ak_raw = ak_src.fetch(ts_code, start=start, end=end, max_news=max_news)
print(" 东方财富(AkShare stock_news_em): {}".format(len(ak_raw)))
except Exception as e:
print(" 东方财富: 获取失败 ({})".format(e))
if use_mcp:
try:
mcp_src = NewsSource(use_akshare=False, use_xwlb=False, use_mcp=True)
mcp_raw = mcp_src.fetch(ts_code, start=start, end=end, max_news=max_news)
print(" MCP(trendradar-news): {}".format(len(mcp_raw)))
except Exception as e:
print(" MCP: 获取失败 ({})".format(e))
all_raw.append((ts_code, xwlb_raw, ak_raw, mcp_raw))
# 合并所有股票的结果
frames = []
for _, x, a, m in all_raw:
for df in [x, a, m]:
if not df.empty:
frames.append(df)
raw_news = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
if not raw_news.empty:
raw_news = raw_news.drop_duplicates(subset=["title", "date"])
raw_news = raw_news.sort_values("date", ascending=False)
print("\n [汇总] 合并去重后: {} 条新闻".format(len(raw_news)))
if raw_news.empty:
print(" (无新闻数据)")
return
src_counts = raw_news["source"].value_counts()
for src, cnt in src_counts.items():
if src == "xwlb":
label = "新闻联播(DB)"
elif src.startswith("akshare"):
label = "东方财富(AkShare)"
elif src.startswith("mcp"):
label = "MCP(trendradar)"
else:
label = src
print(" {}: {}".format(label, cnt))
# ═══════════════════════════════════════════════════════════════
# Step 2: 新闻详情(按来源分开展示)
# ═══════════════════════════════════════════════════════════════
print("\n" + "-" * 72)
print("Step 2: 新闻详情(按数据源分开展示)")
print("-" * 72)
def show_news(label, df, limit=6):
if df.empty:
print("\n [{}] (无数据)".format(label))
return
print("\n [{}] {}".format(label, len(df)))
for i, (_, row) in enumerate(df.head(limit).iterrows()):
title = str(row["title"])[:80]
content_preview = str(row["content"])[:100].replace("\n", " ")
print("\n [{}/{}] {} | {}".format(i + 1, len(df), row["date"], title))
if content_preview:
print(" 内容: {}...".format(content_preview))
url = row.get("url", "")
if url:
print(" 链接: {}".format(url[:100]))
show_news("新闻联播 (xwlb_daily_ext)", raw_news[raw_news["source"] == "xwlb"], limit=6)
show_news("东方财富 (AkShare stock_news_em)",
raw_news[raw_news["source"].str.startswith("akshare")], limit=6)
show_news("MCP (trendradar-news)",
raw_news[raw_news["source"].str.startswith("mcp")], limit=6)
# ═══════════════════════════════════════════════════════════════
# Step 3: 日期对齐
# ═══════════════════════════════════════════════════════════════
print("\n" + "-" * 72)
print("Step 3: 日期对齐到交易日")
print("-" * 72)
# 交易日历:优先用指定股票 DB 缓存,否则 fallback 到 000001.SZ
first_code = ts_codes[0]
price = _get_trading_calendar(dm, first_code)
if price is None:
print(" {} 无 DB 缓存, fallback 到 000001.SZ".format(first_code))
price = _get_trading_calendar(dm, "000001.SZ")
if price is None:
print(" 无交易日历可用")
return
print("\n 交易日历: {} ~ {} ({} 条)".format(price.index[0], price.index[-1], len(price)))
daily_idx = pd.to_datetime(price.index, format="%Y%m%d", errors="coerce")
aligned_news = align_news_to_trading_days(raw_news, daily_idx)
for label, prefix in [("新闻联播", "xwlb"), ("东方财富", "akshare"), ("MCP", "mcp")]:
df = aligned_news[aligned_news["source"].str.startswith(prefix) if prefix != "xwlb"
else (aligned_news["source"] == "xwlb")]
if df.empty:
continue
dates = sorted(df["date"].unique())
print("\n [{}] {} 条 → {} 个交易日 ({})".format(label, len(df), len(dates),
" +1day偏移" if prefix == "xwlb" else " 直接对齐"))
print(" 日期: {} ~ {}".format(dates[0], dates[-1]))
row = df.iloc[0]
print(" 示例: {} | {}...".format(row["date"], str(row["title"])[:60]))
# ═══════════════════════════════════════════════════════════════
# Step 4: Qwen 情绪分析
# ═══════════════════════════════════════════════════════════════
print("\n" + "-" * 72)
print("Step 4: Qwen 情绪分析")
print("-" * 72)
if not has_api:
print("\n [SKIP] Qwen API 跳过 (--no-qwen 或未配置)")
print(" 使用模拟数据演示因子计算逻辑...")
sentiment_results = _mock_sentiment(aligned_news)
else:
max_analyze = min(len(aligned_news), args.max_analyze)
analyze_news = aligned_news.head(max_analyze)
print("\n 逐条分析 {} 条新闻...".format(max_analyze))
sentiment_results = []
for i, (_, row) in enumerate(analyze_news.iterrows()):
title = str(row["title"])
content = str(row["content"]) if len(str(row["content"])) > 20 else ""
text = "{}\n{}".format(title, content)
result = client.analyze_sentiment(text)
sentiment_results.append({
"date": row["date"],
"title": title,
"sentiment_score": result.get("sentiment_score", 0),
"confidence": result.get("confidence", 0),
"impact_duration": result.get("impact_duration", "short"),
"key_topics": json.dumps(result.get("key_topics", [])),
"source": row.get("source", ""),
})
s = result["sentiment_score"]
icon = "(+)" if s > 0.2 else ("(-)" if s < -0.2 else "(o)")
print(" [{}/{}] {} {:+.1f} c={:.2f} | {}...".format(
i + 1, max_analyze, icon, s,
result["confidence"], title[:60]))
sent_df = pd.DataFrame(sentiment_results)
if not sent_df.empty:
print("\n 情绪分析汇总 ({} 条):".format(len(sent_df)))
print(" 平均情绪: {:+.3f}".format(sent_df["sentiment_score"].mean()))
pos = (sent_df["sentiment_score"] > 0.1).sum()
neu = ((sent_df["sentiment_score"] >= -0.1) & (sent_df["sentiment_score"] <= 0.1)).sum()
neg = (sent_df["sentiment_score"] < -0.1).sum()
print(" 正面(>0.1): {} 中性(-0.1~0.1): {} 负面(<-0.1): {}".format(pos, neu, neg))
if "source" in sent_df.columns:
for src in sent_df["source"].unique():
src_df = sent_df[sent_df["source"] == src]
label = src[:20]
print(" [{}] {} 条, 平均情绪: {:+.3f}".format(label, len(src_df), src_df["sentiment_score"].mean()))
# ═══════════════════════════════════════════════════════════════
# Step 5: 因子计算 + 结果输出
# ═══════════════════════════════════════════════════════════════
print("\n" + "-" * 72)
print("Step 5-6: 因子计算 + 结果输出")
print("-" * 72)
from factors.sentiment.sentiment_factor import (
NewsSentimentFactor,
SentimentConfidenceFactor,
SentimentMomentumFactor,
)
if sent_df.empty:
print(" (无情绪数据)")
return
factors = [
NewsSentimentFactor(window=5, decay=0.3, sentiment_df=sent_df),
SentimentConfidenceFactor(window=5, sentiment_df=sent_df),
SentimentMomentumFactor(period=5, sentiment_df=sent_df),
]
factor_results = {}
for f in factors:
series = f.calculate(price)
factor_results[f.name] = series
stats = series.dropna()
if not stats.empty:
print(" {}: mean={:+.4f} std={:.4f} valid={}/{}".format(
f.name, stats.mean(), stats.std(), len(stats), len(series)))
else:
print(" {}: (全NaN)".format(f.name))
factor_df = pd.DataFrame(factor_results)
valid = factor_df.dropna(how="all")
if valid.empty:
print("\n (无有效因子值)")
return
recent = valid.tail(20)
print("\n === 最近 {} 个交易日情绪因子值 ({}) ===".format(len(recent), first_code))
print(" {:<12s} {:>12s} {:>12s} {:>12s}".format("交易日", "news_sent_5", "news_conf_5", "sent_delta_5"))
print(" {} {} {} {}".format("-" * 12, "-" * 12, "-" * 12, "-" * 12))
for idx, row in recent.iterrows():
ns = "{:+.4f}".format(row["news_sent_5"]) if not pd.isna(row["news_sent_5"]) else " NaN"
nc = "{:+.4f}".format(row["news_conf_5"]) if not pd.isna(row["news_conf_5"]) else " NaN"
sd = "{:+.4f}".format(row["sent_delta_5"]) if not pd.isna(row["sent_delta_5"]) else " NaN"
print(" {:<12s} {:>12s} {:>12s} {:>12s}".format(idx, ns, nc, sd))
latest = valid.iloc[-1]
print("\n === 最新交易日 ({}) ===".format(valid.index[-1]))
print(" news_sent_5 : {:+.4f} (加权情绪, >0偏正面)".format(latest["news_sent_5"]))
print(" news_conf_5 : {:+.4f} (置信度加权)".format(latest["news_conf_5"]))
# 情绪贡献明细
print("\n === 情绪贡献明细 (最近3天) ===")
latest_date = valid.index[-1]
nearby = sent_df[
(sent_df["date"] >= str(int(latest_date) - 3)) &
(sent_df["date"] <= latest_date)
]
if not nearby.empty:
for _, row in nearby.head(30).iterrows():
s = row["sentiment_score"]
impact = "(+)" if s > 0.2 else ("(-)" if s < -0.2 else "(o)")
src = str(row.get("source", ""))
src_s = "xwlb" if src == "xwlb" else ("ak" if src.startswith("akshare") else "mcp")
print(" {} [{:+.1f}] [{}] {}...".format(
impact, s, src_s, str(row["title"])[:70]))
else:
print(" (无最近3天新闻)")
try:
from reports.storage import save_report
first = ts_codes[0] if ts_codes else "unknown"
lines = ["## 情绪因子详细演示", "股票: {}".format(", ".join(ts_codes[:5])),
"数据源: {}条新闻".format(len(raw_news)),
"情绪: news_sent_5={}".format(
latest["news_sent_5"] if "news_sent_5" in latest else "N/A")]
save_report("\n".join(lines), "情绪因子详细演示", subject_type="stock", subject_code=first)
print(" 报告已存入 DB")
except Exception:
pass
print("\n" + "=" * 72)
print(" 情绪因子演示完成")
print("=" * 72)
def _get_trading_calendar(dm, ts_code):
"""获取交易日历:优先 DB 缓存;无缓存则尝试 sync_daily 补齐。"""
try:
from database.dao import get_latest_trade_date
if not get_latest_trade_date(ts_code):
print(" {} 无 DB 缓存,尝试 sync_daily 补齐...".format(ts_code))
try:
n = dm.sync_daily(ts_code)
print(" sync_daily 完成: {}".format(n))
except Exception as e:
print(" sync_daily 失败: {}".format(e))
return None
daily = dm.get_daily(ts_code)
if daily is not None and not daily.empty:
daily = daily.set_index("trade_date").sort_index()
if len(daily) > 0:
return daily
except Exception as e:
print(" 获取交易日历异常: {}".format(e))
return None
def _mock_sentiment(news_df):
results = []
for _, row in news_df.iterrows():
title = str(row["title"]).lower()
pos_words = ["利好", "增长", "突破", "创新高", "盈利", "上升", "支持", "回购", "增持", "分红"]
neg_words = ["利空", "下跌", "亏损", "处罚", "减持", "诉讼", "退市", "警告", "暴跌", "违约"]
pos = sum(1 for w in pos_words if w in title)
neg = sum(1 for w in neg_words if w in title)
if pos > neg:
score = min(0.9, 0.1 + pos * 0.2)
elif neg > pos:
score = max(-0.9, -0.1 - neg * 0.2)
else:
score = np.random.uniform(-0.15, 0.15)
results.append({
"date": row["date"], "title": row["title"],
"sentiment_score": round(score, 1),
"confidence": round(np.random.uniform(0.5, 0.9), 2),
"impact_duration": "short", "key_topics": json.dumps([]),
"source": row.get("source", ""),
})
return results
if __name__ == "__main__":
main()