simonandClaude Opus 4.7
271a9343a5
7 Sprints 全部完成: Sprint 0: 基础设施 (DataManager + MariaDB) Sprint 1: 因子引擎 (34因子/12分类) Sprint 2: VectorBT 回测 (5策略+截面) Sprint 3: Optuna 优化 (+Walk-Forward) Sprint 4: ML 模型 (LightGBM+CatBoost) Sprint 5: Qwen 情绪因子 (三源新闻+日期对齐) Sprint 6: Agent 系统 (4Agent+日报.md/.html) 生产加固 (15项): Tushare双源fallback, SSH自动恢复, pool_pre_ping, save_daily先删后插, load_dotenv绝对路径, 日报5d/20d修复, RiskAgent改上证指数, 昨日对比+数据截止, mac_report utf8mb4, CLAUDE-*.md 9条已知Bug, demo全参数化, djapi数据源归一化, indexDatas API修正 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
454 lines
21 KiB
Python
454 lines
21 KiB
Python
"""
|
||
情绪因子详细运行过程演示。
|
||
|
||
用法:
|
||
# 默认：000001.SZ，最近30天，指数范围
|
||
python cli/demo_sentiment_detail.py
|
||
|
||
# 指定股票代码和日期
|
||
python cli/demo_sentiment_detail.py --ts_code 600519.SH --date 20260603
|
||
python cli/demo_sentiment_detail.py --ts_code 000001.SZ,600519.SH,300750.SZ
|
||
|
||
# 指定日期范围
|
||
python cli/demo_sentiment_detail.py --start 20260501 --end 20260603
|
||
|
||
# 分析指定指数成分股
|
||
python cli/demo_sentiment_detail.py --scope-type index --scope-indexes 000300
|
||
|
||
# 分析指定板块
|
||
python cli/demo_sentiment_detail.py --scope-type sector --scope-sectors 银行,电力设备
|
||
|
||
# 只使用特定新闻源
|
||
python cli/demo_sentiment_detail.py --no-xwlb --no-mcp
|
||
python cli/demo_sentiment_detail.py --source akshare
|
||
|
||
# 跳过 Qwen API 调用（仅演示数据流）
|
||
python cli/demo_sentiment_detail.py --no-qwen
|
||
"""
|
||
|
||
import sys, os, json, argparse
|
||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
import pandas as pd
|
||
import numpy as np
|
||
from datetime import datetime
|
||
|
||
|
||
def parse_args():
|
||
p = argparse.ArgumentParser(description="情绪因子详细运行过程演示")
|
||
p.add_argument("--ts_code", default="000001.SZ",
|
||
help="股票代码，多个用逗号分隔（默认: 000001.SZ）")
|
||
p.add_argument("--date", default=None,
|
||
help="目标日期 YYYYMMDD（默认: 今天）")
|
||
p.add_argument("--start", default=None,
|
||
help="起始日期 YYYYMMDD（默认: date-30天）")
|
||
p.add_argument("--end", default=None,
|
||
help="结束日期 YYYYMMDD（默认: date 或今天）")
|
||
p.add_argument("--scope-type", default=None,
|
||
choices=["index", "sector", "custom", "all"],
|
||
help="分析范围类型（覆盖 ts_code）")
|
||
p.add_argument("--scope-indexes", default="000300",
|
||
help="指数代码，逗号分隔（默认: 000300）")
|
||
p.add_argument("--scope-sectors", default="",
|
||
help="板块名称，逗号分隔")
|
||
p.add_argument("--max-news", type=int, default=None,
|
||
help="最大新闻条数（默认: .env SENTIMENT_MAX_NEWS_PER_STOCK 或 30）")
|
||
p.add_argument("--max-analyze", type=int, default=50,
|
||
help="Qwen 分析最大条数（默认: 50，控制成本）")
|
||
p.add_argument("--no-xwlb", action="store_true", help="禁用新闻联播数据源")
|
||
p.add_argument("--no-akshare", action="store_true", help="禁用东方财富数据源")
|
||
p.add_argument("--no-mcp", action="store_true", help="禁用 MCP 数据源")
|
||
p.add_argument("--source", default=None,
|
||
choices=["xwlb", "akshare", "mcp"],
|
||
help="仅使用指定数据源")
|
||
p.add_argument("--no-qwen", action="store_true", help="跳过 Qwen 分析（仅演示数据流）")
|
||
return p.parse_args()
|
||
|
||
|
||
def main():
|
||
args = parse_args()
|
||
|
||
date = args.date or datetime.now().strftime("%Y%m%d")
|
||
start = args.start or (datetime.strptime(date, "%Y%m%d") - pd.Timedelta(days=30)).strftime("%Y%m%d")
|
||
end = args.end or date
|
||
|
||
print("=" * 72)
|
||
print(" 情绪因子详细运行过程")
|
||
print("=" * 72)
|
||
print(" 日期: {} ~ {} (目标: {})".format(start, end, date))
|
||
|
||
# ═══════════════════════════════════════════════════════════════
|
||
# Step 0: 初始化
|
||
# ═══════════════════════════════════════════════════════════════
|
||
print("\n" + "-" * 72)
|
||
print("Step 0: 初始化引擎")
|
||
print("-" * 72)
|
||
|
||
from data.data_manager import DataManager
|
||
from factors.sentiment.qwen_client import QwenClient
|
||
from factors.sentiment.news_source import NewsSource, align_news_to_trading_days
|
||
|
||
dm = DataManager()
|
||
dm.init_db()
|
||
client = QwenClient()
|
||
|
||
has_api = (bool(client.api_key) or bool(client.local_base_url)) and not args.no_qwen
|
||
use_xwlb = not args.no_xwlb and (args.source is None or args.source == "xwlb")
|
||
use_akshare = not args.no_akshare and (args.source is None or args.source == "akshare")
|
||
use_mcp = not args.no_mcp and (args.source is None or args.source == "mcp")
|
||
|
||
print(" Qwen API: {}".format("DashScope/{}".format(client.model) if (has_api and not client.local_base_url) else (
|
||
"本地 Ollama/{}".format(client.local_model) if (has_api and client.local_base_url) else "跳过（--no-qwen 或未配置）")))
|
||
print(" 数据源: {}/{}/{}".format(
|
||
"xwlb" if use_xwlb else "xwlb(off)",
|
||
"akshare" if use_akshare else "akshare(off)",
|
||
"mcp" if use_mcp else "mcp(off)",
|
||
))
|
||
max_news = args.max_news or int(os.getenv("SENTIMENT_MAX_NEWS_PER_STOCK", "30"))
|
||
print(" 最大新闻: {} 条 (SENTIMENT_MAX_NEWS_PER_STOCK={})".format(
|
||
max_news, os.getenv("SENTIMENT_MAX_NEWS_PER_STOCK", "未设置")))
|
||
|
||
# 分析范围
|
||
if args.scope_type:
|
||
from factors.sentiment.sentiment_engine import SentimentEngine
|
||
# 临时覆盖环境变量
|
||
os.environ["SENTIMENT_SCOPE_TYPE"] = args.scope_type
|
||
if args.scope_indexes:
|
||
os.environ["SENTIMENT_SCOPE_INDEXES"] = args.scope_indexes
|
||
if args.scope_sectors:
|
||
os.environ["SENTIMENT_SCOPE_SECTORS"] = args.scope_sectors
|
||
sent_tmp = SentimentEngine(dm)
|
||
ts_codes = sent_tmp.get_scope_stocks()
|
||
print(" 分析范围: {} ({})".format(args.scope_type, len(ts_codes)))
|
||
if len(ts_codes) > 10:
|
||
print(" 股票示例: {}... (共 {} 只)".format(", ".join(ts_codes[:10]), len(ts_codes)))
|
||
else:
|
||
print(" 股票: {}".format(", ".join(ts_codes)))
|
||
else:
|
||
ts_codes = [c.strip() for c in args.ts_code.split(",") if c.strip()]
|
||
|
||
# ═══════════════════════════════════════════════════════════════
|
||
# Step 1: 分别从三个数据源获取新闻
|
||
# ═══════════════════════════════════════════════════════════════
|
||
print("\n" + "-" * 72)
|
||
print("Step 1: 获取新闻 ({} 只股票)".format(len(ts_codes)))
|
||
print("-" * 72)
|
||
|
||
all_raw = []
|
||
for ts_code in ts_codes:
|
||
print("\n --- {} ---".format(ts_code))
|
||
|
||
xwlb_raw = pd.DataFrame()
|
||
ak_raw = pd.DataFrame()
|
||
mcp_raw = pd.DataFrame()
|
||
|
||
if use_xwlb:
|
||
try:
|
||
xwlb_src = NewsSource(use_akshare=False, use_mcp=False)
|
||
xwlb_raw = xwlb_src.fetch(ts_code, start=start, end=end, max_news=max_news * 3)
|
||
print(" 新闻联播(DB xwlb_daily_ext): {} 条 (news_date范围: {}-1~{}-1)".format(
|
||
len(xwlb_raw), start, end))
|
||
except Exception as e:
|
||
print(" 新闻联播: 获取失败 ({})".format(e))
|
||
|
||
if use_akshare:
|
||
try:
|
||
ak_src = NewsSource(use_xwlb=False, use_mcp=False)
|
||
ak_raw = ak_src.fetch(ts_code, start=start, end=end, max_news=max_news)
|
||
print(" 东方财富(AkShare stock_news_em): {} 条".format(len(ak_raw)))
|
||
except Exception as e:
|
||
print(" 东方财富: 获取失败 ({})".format(e))
|
||
|
||
if use_mcp:
|
||
try:
|
||
mcp_src = NewsSource(use_akshare=False, use_xwlb=False, use_mcp=True)
|
||
mcp_raw = mcp_src.fetch(ts_code, start=start, end=end, max_news=max_news)
|
||
print(" MCP(trendradar-news): {} 条".format(len(mcp_raw)))
|
||
except Exception as e:
|
||
print(" MCP: 获取失败 ({})".format(e))
|
||
|
||
all_raw.append((ts_code, xwlb_raw, ak_raw, mcp_raw))
|
||
|
||
# 合并所有股票的结果
|
||
frames = []
|
||
for _, x, a, m in all_raw:
|
||
for df in [x, a, m]:
|
||
if not df.empty:
|
||
frames.append(df)
|
||
raw_news = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
|
||
|
||
if not raw_news.empty:
|
||
raw_news = raw_news.drop_duplicates(subset=["title", "date"])
|
||
raw_news = raw_news.sort_values("date", ascending=False)
|
||
|
||
print("\n [汇总] 合并去重后: {} 条新闻".format(len(raw_news)))
|
||
if raw_news.empty:
|
||
print(" (无新闻数据)")
|
||
return
|
||
|
||
src_counts = raw_news["source"].value_counts()
|
||
for src, cnt in src_counts.items():
|
||
if src == "xwlb":
|
||
label = "新闻联播(DB)"
|
||
elif src.startswith("akshare"):
|
||
label = "东方财富(AkShare)"
|
||
elif src.startswith("mcp"):
|
||
label = "MCP(trendradar)"
|
||
else:
|
||
label = src
|
||
print(" {}: {} 条".format(label, cnt))
|
||
|
||
# ═══════════════════════════════════════════════════════════════
|
||
# Step 2: 新闻详情（按来源分开展示）
|
||
# ═══════════════════════════════════════════════════════════════
|
||
print("\n" + "-" * 72)
|
||
print("Step 2: 新闻详情（按数据源分开展示）")
|
||
print("-" * 72)
|
||
|
||
def show_news(label, df, limit=6):
|
||
if df.empty:
|
||
print("\n [{}] (无数据)".format(label))
|
||
return
|
||
print("\n [{}] {} 条".format(label, len(df)))
|
||
for i, (_, row) in enumerate(df.head(limit).iterrows()):
|
||
title = str(row["title"])[:80]
|
||
content_preview = str(row["content"])[:100].replace("\n", " ")
|
||
print("\n [{}/{}] {} | {}".format(i + 1, len(df), row["date"], title))
|
||
if content_preview:
|
||
print(" 内容: {}...".format(content_preview))
|
||
url = row.get("url", "")
|
||
if url:
|
||
print(" 链接: {}".format(url[:100]))
|
||
|
||
show_news("新闻联播 (xwlb_daily_ext)", raw_news[raw_news["source"] == "xwlb"], limit=6)
|
||
show_news("东方财富 (AkShare stock_news_em)",
|
||
raw_news[raw_news["source"].str.startswith("akshare")], limit=6)
|
||
show_news("MCP (trendradar-news)",
|
||
raw_news[raw_news["source"].str.startswith("mcp")], limit=6)
|
||
|
||
# ═══════════════════════════════════════════════════════════════
|
||
# Step 3: 日期对齐
|
||
# ═══════════════════════════════════════════════════════════════
|
||
print("\n" + "-" * 72)
|
||
print("Step 3: 日期对齐到交易日")
|
||
print("-" * 72)
|
||
|
||
# 交易日历：优先用指定股票 DB 缓存，否则 fallback 到 000001.SZ
|
||
first_code = ts_codes[0]
|
||
price = _get_trading_calendar(dm, first_code)
|
||
if price is None:
|
||
print(" {} 无 DB 缓存, fallback 到 000001.SZ".format(first_code))
|
||
price = _get_trading_calendar(dm, "000001.SZ")
|
||
if price is None:
|
||
print(" 无交易日历可用")
|
||
return
|
||
print("\n 交易日历: {} ~ {} ({} 条)".format(price.index[0], price.index[-1], len(price)))
|
||
|
||
daily_idx = pd.to_datetime(price.index, format="%Y%m%d", errors="coerce")
|
||
aligned_news = align_news_to_trading_days(raw_news, daily_idx)
|
||
|
||
for label, prefix in [("新闻联播", "xwlb"), ("东方财富", "akshare"), ("MCP", "mcp")]:
|
||
df = aligned_news[aligned_news["source"].str.startswith(prefix) if prefix != "xwlb"
|
||
else (aligned_news["source"] == "xwlb")]
|
||
if df.empty:
|
||
continue
|
||
dates = sorted(df["date"].unique())
|
||
print("\n [{}] {} 条 → {} 个交易日 ({})".format(label, len(df), len(dates),
|
||
" +1day偏移" if prefix == "xwlb" else " 直接对齐"))
|
||
print(" 日期: {} ~ {}".format(dates[0], dates[-1]))
|
||
row = df.iloc[0]
|
||
print(" 示例: {} | {}...".format(row["date"], str(row["title"])[:60]))
|
||
|
||
# ═══════════════════════════════════════════════════════════════
|
||
# Step 4: Qwen 情绪分析
|
||
# ═══════════════════════════════════════════════════════════════
|
||
print("\n" + "-" * 72)
|
||
print("Step 4: Qwen 情绪分析")
|
||
print("-" * 72)
|
||
|
||
if not has_api:
|
||
print("\n [SKIP] Qwen API 跳过 (--no-qwen 或未配置)")
|
||
print(" 使用模拟数据演示因子计算逻辑...")
|
||
sentiment_results = _mock_sentiment(aligned_news)
|
||
else:
|
||
max_analyze = min(len(aligned_news), args.max_analyze)
|
||
analyze_news = aligned_news.head(max_analyze)
|
||
|
||
print("\n 逐条分析 {} 条新闻...".format(max_analyze))
|
||
sentiment_results = []
|
||
for i, (_, row) in enumerate(analyze_news.iterrows()):
|
||
title = str(row["title"])
|
||
content = str(row["content"]) if len(str(row["content"])) > 20 else ""
|
||
text = "{}\n{}".format(title, content)
|
||
|
||
result = client.analyze_sentiment(text)
|
||
sentiment_results.append({
|
||
"date": row["date"],
|
||
"title": title,
|
||
"sentiment_score": result.get("sentiment_score", 0),
|
||
"confidence": result.get("confidence", 0),
|
||
"impact_duration": result.get("impact_duration", "short"),
|
||
"key_topics": json.dumps(result.get("key_topics", [])),
|
||
"source": row.get("source", ""),
|
||
})
|
||
|
||
s = result["sentiment_score"]
|
||
icon = "(+)" if s > 0.2 else ("(-)" if s < -0.2 else "(o)")
|
||
print(" [{}/{}] {} {:+.1f} c={:.2f} | {}...".format(
|
||
i + 1, max_analyze, icon, s,
|
||
result["confidence"], title[:60]))
|
||
|
||
sent_df = pd.DataFrame(sentiment_results)
|
||
if not sent_df.empty:
|
||
print("\n 情绪分析汇总 ({} 条):".format(len(sent_df)))
|
||
print(" 平均情绪: {:+.3f}".format(sent_df["sentiment_score"].mean()))
|
||
pos = (sent_df["sentiment_score"] > 0.1).sum()
|
||
neu = ((sent_df["sentiment_score"] >= -0.1) & (sent_df["sentiment_score"] <= 0.1)).sum()
|
||
neg = (sent_df["sentiment_score"] < -0.1).sum()
|
||
print(" 正面(>0.1): {} 中性(-0.1~0.1): {} 负面(<-0.1): {}".format(pos, neu, neg))
|
||
if "source" in sent_df.columns:
|
||
for src in sent_df["source"].unique():
|
||
src_df = sent_df[sent_df["source"] == src]
|
||
label = src[:20]
|
||
print(" [{}] {} 条, 平均情绪: {:+.3f}".format(label, len(src_df), src_df["sentiment_score"].mean()))
|
||
|
||
# ═══════════════════════════════════════════════════════════════
|
||
# Step 5: 因子计算 + 结果输出
|
||
# ═══════════════════════════════════════════════════════════════
|
||
print("\n" + "-" * 72)
|
||
print("Step 5-6: 因子计算 + 结果输出")
|
||
print("-" * 72)
|
||
|
||
from factors.sentiment.sentiment_factor import (
|
||
NewsSentimentFactor,
|
||
SentimentConfidenceFactor,
|
||
SentimentMomentumFactor,
|
||
)
|
||
|
||
if sent_df.empty:
|
||
print(" (无情绪数据)")
|
||
return
|
||
|
||
factors = [
|
||
NewsSentimentFactor(window=5, decay=0.3, sentiment_df=sent_df),
|
||
SentimentConfidenceFactor(window=5, sentiment_df=sent_df),
|
||
SentimentMomentumFactor(period=5, sentiment_df=sent_df),
|
||
]
|
||
|
||
factor_results = {}
|
||
for f in factors:
|
||
series = f.calculate(price)
|
||
factor_results[f.name] = series
|
||
stats = series.dropna()
|
||
if not stats.empty:
|
||
print(" {}: mean={:+.4f} std={:.4f} valid={}/{}".format(
|
||
f.name, stats.mean(), stats.std(), len(stats), len(series)))
|
||
else:
|
||
print(" {}: (全NaN)".format(f.name))
|
||
|
||
factor_df = pd.DataFrame(factor_results)
|
||
valid = factor_df.dropna(how="all")
|
||
|
||
if valid.empty:
|
||
print("\n (无有效因子值)")
|
||
return
|
||
|
||
recent = valid.tail(20)
|
||
print("\n === 最近 {} 个交易日情绪因子值 ({}) ===".format(len(recent), first_code))
|
||
print(" {:<12s} {:>12s} {:>12s} {:>12s}".format("交易日", "news_sent_5", "news_conf_5", "sent_delta_5"))
|
||
print(" {} {} {} {}".format("-" * 12, "-" * 12, "-" * 12, "-" * 12))
|
||
for idx, row in recent.iterrows():
|
||
ns = "{:+.4f}".format(row["news_sent_5"]) if not pd.isna(row["news_sent_5"]) else " NaN"
|
||
nc = "{:+.4f}".format(row["news_conf_5"]) if not pd.isna(row["news_conf_5"]) else " NaN"
|
||
sd = "{:+.4f}".format(row["sent_delta_5"]) if not pd.isna(row["sent_delta_5"]) else " NaN"
|
||
print(" {:<12s} {:>12s} {:>12s} {:>12s}".format(idx, ns, nc, sd))
|
||
|
||
latest = valid.iloc[-1]
|
||
print("\n === 最新交易日 ({}) ===".format(valid.index[-1]))
|
||
print(" news_sent_5 : {:+.4f} (加权情绪, >0偏正面)".format(latest["news_sent_5"]))
|
||
print(" news_conf_5 : {:+.4f} (置信度加权)".format(latest["news_conf_5"]))
|
||
|
||
# 情绪贡献明细
|
||
print("\n === 情绪贡献明细 (最近3天) ===")
|
||
latest_date = valid.index[-1]
|
||
nearby = sent_df[
|
||
(sent_df["date"] >= str(int(latest_date) - 3)) &
|
||
(sent_df["date"] <= latest_date)
|
||
]
|
||
if not nearby.empty:
|
||
for _, row in nearby.head(30).iterrows():
|
||
s = row["sentiment_score"]
|
||
impact = "(+)" if s > 0.2 else ("(-)" if s < -0.2 else "(o)")
|
||
src = str(row.get("source", ""))
|
||
src_s = "xwlb" if src == "xwlb" else ("ak" if src.startswith("akshare") else "mcp")
|
||
print(" {} [{:+.1f}] [{}] {}...".format(
|
||
impact, s, src_s, str(row["title"])[:70]))
|
||
else:
|
||
print(" (无最近3天新闻)")
|
||
|
||
try:
|
||
from reports.storage import save_report
|
||
first = ts_codes[0] if ts_codes else "unknown"
|
||
lines = ["## 情绪因子详细演示", "股票: {}".format(", ".join(ts_codes[:5])),
|
||
"数据源: {}条新闻".format(len(raw_news)),
|
||
"情绪: news_sent_5={}".format(
|
||
latest["news_sent_5"] if "news_sent_5" in latest else "N/A")]
|
||
save_report("\n".join(lines), "情绪因子详细演示", subject_type="stock", subject_code=first)
|
||
print(" 报告已存入 DB")
|
||
except Exception:
|
||
pass
|
||
|
||
print("\n" + "=" * 72)
|
||
print(" 情绪因子演示完成")
|
||
print("=" * 72)
|
||
|
||
|
||
def _get_trading_calendar(dm, ts_code):
|
||
"""获取交易日历：优先 DB 缓存；无缓存则尝试 sync_daily 补齐。"""
|
||
try:
|
||
from database.dao import get_latest_trade_date
|
||
if not get_latest_trade_date(ts_code):
|
||
print(" {} 无 DB 缓存，尝试 sync_daily 补齐...".format(ts_code))
|
||
try:
|
||
n = dm.sync_daily(ts_code)
|
||
print(" sync_daily 完成: {} 条".format(n))
|
||
except Exception as e:
|
||
print(" sync_daily 失败: {}".format(e))
|
||
return None
|
||
daily = dm.get_daily(ts_code)
|
||
if daily is not None and not daily.empty:
|
||
daily = daily.set_index("trade_date").sort_index()
|
||
if len(daily) > 0:
|
||
return daily
|
||
except Exception as e:
|
||
print(" 获取交易日历异常: {}".format(e))
|
||
return None
|
||
|
||
|
||
def _mock_sentiment(news_df):
|
||
results = []
|
||
for _, row in news_df.iterrows():
|
||
title = str(row["title"]).lower()
|
||
pos_words = ["利好", "增长", "突破", "创新高", "盈利", "上升", "支持", "回购", "增持", "分红"]
|
||
neg_words = ["利空", "下跌", "亏损", "处罚", "减持", "诉讼", "退市", "警告", "暴跌", "违约"]
|
||
pos = sum(1 for w in pos_words if w in title)
|
||
neg = sum(1 for w in neg_words if w in title)
|
||
if pos > neg:
|
||
score = min(0.9, 0.1 + pos * 0.2)
|
||
elif neg > pos:
|
||
score = max(-0.9, -0.1 - neg * 0.2)
|
||
else:
|
||
score = np.random.uniform(-0.15, 0.15)
|
||
results.append({
|
||
"date": row["date"], "title": row["title"],
|
||
"sentiment_score": round(score, 1),
|
||
"confidence": round(np.random.uniform(0.5, 0.9), 2),
|
||
"impact_duration": "short", "key_topics": json.dumps([]),
|
||
"source": row.get("source", ""),
|
||
})
|
||
return results
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|