"""个股日报生成器 v2.0。
根据 watchlist.yaml 配置,为每只关注股票生成日报:
- AI 要点分析(DeepSeek 生成)
- 公告 / 调研 / 互动问答(cninfo v2.0 CninfoItem 数据)
- 相关新闻(Qdrant 语义检索)
- HTML 报告 + 自动上传
数据来源:
- cninfo 数据: data/raw/cninfo/{YYYYMMDD}/*.json (CninfoItem v2.0 格式)
- 新闻: Qdrant 向量检索
"""
from __future__ import annotations
import json
import os as _os
import re
import subprocess
import time as _time
from datetime import date, datetime, timedelta
from pathlib import Path
from typing import Any
from loguru import logger
UPLOAD_HOST = "simon@doorcome.cn"
UPLOAD_BASE = "/var/www/html/echart/research"
# --------------------------------------------------------------------------- #
# 配置
# --------------------------------------------------------------------------- #
def _load_source_names() -> dict[str, str]:
import yaml
try:
with open("configs/sources.yaml", encoding="utf-8") as f:
data = yaml.safe_load(f)
names = {s["id"]: s["name"] for s in (data.get("sources") or []) if s.get("id")}
except Exception:
names = {}
names["cninfo"] = "巨潮资讯网"
return names
_SOURCE_NAMES = _load_source_names()
def _load_watchlist() -> list[dict]:
import yaml
try:
with open("configs/watchlist.yaml", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
return list(data.get("watchlist") or [])
except Exception:
return []
def _report_days() -> int:
"""从 .env 读取报告天数,默认 15。"""
return int(_os.environ.get("STOCK_REPORT_DAYS", "15"))
# --------------------------------------------------------------------------- #
# cninfo 数据读取 (v2.0 CninfoItem 格式)
# --------------------------------------------------------------------------- #
def _read_cninfo_items(
code: str,
days_back: int | None = None,
item_type: str | None = None,
) -> list[dict]:
"""从 data/raw/cninfo/ 中读取指定股票的 CninfoItem JSON 数据。
Args:
code: 6 位股票代码
days_back: 向前追溯天数,为 None 则使用 STOCK_REPORT_DAYS
item_type: 过滤类型 None=全部, announcement/research/irm
"""
if days_back is None:
days_back = _report_days()
since = date.today() - timedelta(days=days_back)
since_str = since.strftime("%Y-%m-%d")
raw_root = Path("data/raw/cninfo")
if not raw_root.is_dir():
logger.warning("cninfo raw 目录不存在: {}", raw_root)
return []
items: list[dict] = []
for day_dir in sorted(raw_root.glob("*"), reverse=True):
# 解析目录日期（抓取日期），用于早期跳出循环
try:
day_str = day_dir.name
if len(day_str) != 8:
continue
# 目录日期仅用于性能优化：如果目录日期太旧(>days_back*2)，跳过
day_date = date(int(day_str[:4]), int(day_str[4:6]), int(day_str[6:]))
if day_date < since - timedelta(days=days_back):
continue
except ValueError:
continue
# 从此日期的 index.jsonl 读取
idx_path = day_dir / "index.jsonl"
if not idx_path.is_file():
# 直接读取 JSON 文件
for jf in sorted(day_dir.glob("*.json"), reverse=True):
item = _try_load_cninfo_item(jf, code, item_type)
if item:
pt = (item.get("publish_time") or "").strip()
if pt and pt >= since_str:
items.append(item)
elif not pt:
# publish_time 为空（如 irm），仍纳入但标记
items.append(item)
else:
with idx_path.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
rec = json.loads(line)
except json.JSONDecodeError:
continue
# 按股票和类型过滤
if rec.get("stock_code") != code:
continue
if item_type and rec.get("item_type") != item_type:
continue
pt = (rec.get("publish_time") or "").strip()
# 按 publish_time 过滤（而非抓取日期）
if pt and pt < since_str:
continue
items.append({
"title": (rec.get("title") or "").strip(),
"url": (rec.get("url") or "").strip(),
"source": "巨潮资讯网",
"score": 1.0,
"publish_time": pt,
"item_type": (rec.get("item_type") or "").strip(),
"event": rec.get("extra", {}),
})
# 限制同一天/同一类型最多取 50 条
if len(items) >= 50:
break
return items
def _try_load_cninfo_item(json_path: Path, code: str,
item_type: str | None) -> dict | None:
"""从单个 CninfoItem JSON 文件加载(无 index.jsonl 时的回退)。"""
try:
data = json.loads(json_path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return None
if data.get("stock_code") != code:
return None
if item_type and data.get("item_type") != item_type:
return None
return {
"title": (data.get("title") or "").strip(),
"url": (data.get("url") or "").strip(),
"source": "巨潮资讯网",
"score": 1.0,
"publish_time": (data.get("publish_time") or "").strip(),
"item_type": (data.get("item_type") or "").strip(),
"event": data.get("extra", {}),
}
# --------------------------------------------------------------------------- #
# Qdrant 新闻搜索
# --------------------------------------------------------------------------- #
def _search_news_from_qdrant(
emb: Any, store: Any, query_text: str,
stock_codes: list[str], top_k: int = 30,
days_back: int | None = None, company_name: str = "",
) -> list[dict]:
"""多策略搜索 Qdrant: 股票代码 → 公司名 → 语义。"""
from vectorstore import SearchFilter
if days_back is None:
days_back = _report_days()
vec = emb.embed_one(query_text)
since = (date.today() - timedelta(days=days_back)).strftime("%Y-%m-%d")
# 补全后缀
codes_with_suffix = []
for c in stock_codes:
codes_with_suffix.extend([f"{c}.SZ", f"{c}.SH", f"{c}.BJ", c])
hits: list = []
hits = store.query(query_vector=vec, top_k=top_k,
filter=SearchFilter(stock_codes=codes_with_suffix,
publish_date_from=since))
if not hits and company_name:
hits = store.query(query_vector=vec, top_k=top_k,
filter=SearchFilter(company_names=[company_name],
publish_date_from=since))
if not hits:
hits = store.query(query_vector=vec, top_k=top_k,
filter=SearchFilter(publish_date_from=since))
return [
{
"title": h.title, "url": h.url,
"source": _SOURCE_NAMES.get(h.source_id, h.source_id),
"score": round(h.score, 4),
"publish_time": h.publish_time.isoformat() if h.publish_time else None,
"event": h.event or {},
}
for h in hits
if _SOURCE_NAMES.get(h.source_id, h.source_id) != "巨潮资讯网"
]
# --------------------------------------------------------------------------- #
# LLM AI 要点分析
# --------------------------------------------------------------------------- #
def _generate_ai_summary(company_name: str, announcements: list[dict],
news: list[dict], research: list[dict],
irm: list[dict]) -> str:
"""LLM 生成个股要点分析。"""
from llm.client import load_llm_config, make_sync_client
lines = []
if announcements:
lines.append(f"## 公告 ({len(announcements)} 条)")
for a in announcements[:10]:
lines.append(f"- {a['title']} ({a.get('publish_time', '')})")
if research:
lines.append(f"## 调研 ({len(research)} 条)")
for r in research[:5]:
lines.append(f"- {r['title']} ({r.get('publish_time', '')})")
if news:
lines.append(f"## 新闻 ({len(news)} 条)")
for n in news[:10]:
ev = n.get("event", {})
summary = ev.get("summary", "")
lines.append(
f"- [{n['source']}] {n['title']}"
+ (f"。{summary}" if summary else "")
)
if irm:
lines.append(f"## 互动问答 ({len(irm)} 条)")
for q in irm[:5]:
lines.append(f"- {q['title']}")
if not lines:
return "暂无足够数据生成 AI 摘要"
prompt = f"""以下是 {company_name} 近期的公告、调研、新闻和互动问答:
{chr(10).join(lines)[:3500]}
请输出 5-8 条要点分析,每条以 "- " 开头:
1. 最重要的公告或事件是什么?影响如何?
2. 近期有哪些值得关注的动态?
3. 市场情绪倾向(利好/利空)?
4. 后续需要关注什么?
直接输出要点列表:"""
try:
config = load_llm_config()
client = make_sync_client(config)
resp = client.chat.completions.create(
model=config.model,
messages=[{"role": "user", "content": prompt}],
temperature=0.3, max_tokens=500,
)
return (resp.choices[0].message.content or "").strip()
except Exception as e:
logger.warning("个股 AI 摘要失败: {}", e)
return "AI 摘要暂不可用"
# --------------------------------------------------------------------------- #
# HTML 渲染
# --------------------------------------------------------------------------- #
def _clean_markdown(text: str) -> str:
"""LLM 输出的简单 Markdown 转 HTML 片段。"""
text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
text = re.sub(r"\*(.+?)\*", r"\1", text)
text = re.sub(r"`(.+?)`", r"\1", text)
return text
_HTML_TEMPLATE = """
{company_name}({stock_code}) 个股日报 — {report_date}
一、AI 要点分析
{ai_summary_html}
二、公司公告 ({ann_count} 条)
近 {report_days} 日公告,来源 巨潮资讯网
{ann_table}
三、调研活动 ({research_count} 条)
近 {report_days} 日投资者关系活动,来源 巨潮资讯网
{research_table}
四、相关新闻 ({news_count} 条)
近 {report_days} 日财经新闻
{news_table}
五、互动问答 ({irm_count} 条)
近 {report_days} 日互动易平台问答
{irm_table}
"""
def _render_table(items: list[dict], max_rows: int = 10) -> str:
if not items:
return "暂无数据
"
rows = []
for i, item in enumerate(items[:max_rows], 1):
ev = item.get("event", {})
sentiment = ev.get("sentiment", "")
badge = {"positive": "badge-pos", "negative": "badge-neg"}.get(sentiment, "badge-neu")
icon = {"positive": "🟢", "negative": "🔴", "neutral": "⚪"}.get(sentiment, "")
short_title = item["title"][:60]
if len(item["title"]) > 60:
short_title += "..."
date_str = (item.get("publish_time") or "")[:10]
url = item.get("url", "")
title_cell = (
f'{short_title}'
if url else short_title
)
rows.append(
f'| {i} | '
f'{icon} | '
f'{title_cell} | '
f'{item["source"]} | '
f'{date_str} |
'
)
return (
f"| # | | 标题 | 来源 | 日期 |
|---|
"
f"{''.join(rows)}
"
)
# --------------------------------------------------------------------------- #
# 单股报告生成
# --------------------------------------------------------------------------- #
def _generate_stock_report_with_backend(stock: dict, store: Any, emb: Any, *,
upload: bool = True) -> Path | None:
"""为单个股票生成日报(使用共享 Qdrant backend)。"""
code = stock["code"]
name = stock["name"]
days = _report_days()
logger.info("生成个股报告: {} ({}) 近{}日", code, name, days)
# ---- 从 cninfo v2.0 数据读取 ----
ann_items = _read_cninfo_items(code, days_back=days, item_type="announcement")
research_items = _read_cninfo_items(code, days_back=days, item_type="research")
irm_items = _read_cninfo_items(code, days_back=days, item_type="irm")
# ---- 新闻: Qdrant 语义检索 ----
news_items = _search_news_from_qdrant(
emb, store, f"{name} {code}", stock_codes=[code],
top_k=30, days_back=days, company_name=name,
)
# ---- AI 摘要 ----
ai = _generate_ai_summary(name, ann_items, news_items, research_items, irm_items)
ai = _clean_markdown(ai)
ai_html = (
"" + "".join(
f"- {ln[2:]}
" if ln.startswith("- ") else f"- {ln}
"
for ln in ai.strip().splitlines() if ln.strip()
) + "
"
if ai else "AI 摘要暂不可用
"
)
# ---- 渲染 HTML ----
today = date.today()
start_date = today - timedelta(days=days)
html = _HTML_TEMPLATE.format(
company_name=name, stock_code=code,
report_date=today.strftime("%Y-%m-%d"),
date_from=start_date.strftime("%Y-%m-%d"),
date_to=today.strftime("%Y-%m-%d"),
generated_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
report_days=days,
ai_summary_html=ai_html,
ann_count=len(ann_items),
ann_table=_render_table(ann_items),
research_count=len(research_items),
research_table=_render_table(research_items, max_rows=10),
news_count=len(news_items),
news_table=_render_table(news_items),
irm_count=len(irm_items),
irm_table=_render_table(irm_items, max_rows=10),
)
# ---- 保存 ----
out_dir = Path("data/reports/stocks")
out_dir.mkdir(parents=True, exist_ok=True)
fname = f"{code}_{name}_个股日报_{today.strftime('%Y%m%d')}.html"
html_path = out_dir / fname
html_path.write_text(html, encoding="utf-8")
logger.info("个股报告已保存: {} ({} KB)", html_path, len(html) // 1024)
# ---- 上传 ----
if upload:
_upload_stock_report(html_path, today.strftime("%Y%m%d"))
return html_path
# --------------------------------------------------------------------------- #
# 上传
# --------------------------------------------------------------------------- #
def _upload_stock_report(html_path: Path, day_str: str) -> bool:
"""上传个股报告到 Web 服务器。"""
remote_dir = f"{UPLOAD_BASE}/{day_str}/"
try:
r1 = subprocess.run(
["ssh", UPLOAD_HOST, f"mkdir -p {remote_dir}"],
timeout=15, capture_output=True, text=True,
)
r2 = subprocess.run(
["scp", str(html_path), f"{UPLOAD_HOST}:{remote_dir}{html_path.name}"],
timeout=30, capture_output=True, text=True,
)
return r1.returncode == 0 and r2.returncode == 0
except Exception:
return False
# --------------------------------------------------------------------------- #
# 批量生成主入口
# --------------------------------------------------------------------------- #
def generate_all_stock_reports(upload: bool = True) -> int:
"""为关注列表中所有股票生成个股日报。返回生成的报告数。"""
watchlist = _load_watchlist()
if not watchlist:
logger.warning("关注列表为空,跳过个股报告")
return 0
# 共享 backend(Qdrant + Embedder, 加锁重试)
from dotenv import load_dotenv
load_dotenv()
from embedding import make_sync_provider
from vectorstore import VectorStore, make_qdrant_client
emb = make_sync_provider()
for retry in range(5):
try:
client = make_qdrant_client()
store = VectorStore(client)
break
except RuntimeError:
if retry < 4:
logger.warning("Qdrant 被占用,{} 秒后重试...", (retry + 1) * 2)
_time.sleep((retry + 1) * 2)
else:
raise
count = 0
for stock in watchlist:
code = stock.get("code", "")
name = stock.get("name", "")
try:
path = _generate_stock_report_with_backend(stock, store, emb, upload=upload)
if path:
count += 1
except Exception as e:
logger.error("个股报告生成失败 {} {}: {}", code, name, e)
store.close()
emb.close()
logger.info("个股报告完成: {}/{} 家", count, len(watchlist))
return count