535 lines
20 KiB
Python
535 lines
20 KiB
Python
"""个股日报生成器 v2.0。
|
|
|
|
根据 watchlist.yaml 配置,为每只关注股票生成日报:
|
|
- AI 要点分析(DeepSeek 生成)
|
|
- 公告 / 调研 / 互动问答(cninfo v2.0 CninfoItem 数据)
|
|
- 相关新闻(Qdrant 语义检索)
|
|
- HTML 报告 + 自动上传
|
|
|
|
数据来源:
|
|
- cninfo 数据: data/raw/cninfo/{YYYYMMDD}/*.json (CninfoItem v2.0 格式)
|
|
- 新闻: Qdrant 向量检索
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os as _os
|
|
import re
|
|
import subprocess
|
|
import time as _time
|
|
from datetime import date, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from loguru import logger
|
|
|
|
UPLOAD_HOST = "simon@doorcome.cn"
|
|
UPLOAD_BASE = "/var/www/html/echart/research"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 配置
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def _load_source_names() -> dict[str, str]:
|
|
import yaml
|
|
try:
|
|
with open("configs/sources.yaml", encoding="utf-8") as f:
|
|
data = yaml.safe_load(f)
|
|
names = {s["id"]: s["name"] for s in (data.get("sources") or []) if s.get("id")}
|
|
except Exception:
|
|
names = {}
|
|
names["cninfo"] = "巨潮资讯网"
|
|
return names
|
|
|
|
|
|
_SOURCE_NAMES = _load_source_names()
|
|
|
|
|
|
def _load_watchlist() -> list[dict]:
|
|
import yaml
|
|
try:
|
|
with open("configs/watchlist.yaml", encoding="utf-8") as f:
|
|
data = yaml.safe_load(f) or {}
|
|
return list(data.get("watchlist") or [])
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _report_days() -> int:
|
|
"""从 .env 读取报告天数,默认 15。"""
|
|
return int(_os.environ.get("STOCK_REPORT_DAYS", "15"))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# cninfo 数据读取 (v2.0 CninfoItem 格式)
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def _read_cninfo_items(
|
|
code: str,
|
|
days_back: int | None = None,
|
|
item_type: str | None = None,
|
|
) -> list[dict]:
|
|
"""从 data/raw/cninfo/ 中读取指定股票的 CninfoItem JSON 数据。
|
|
|
|
Args:
|
|
code: 6 位股票代码
|
|
days_back: 向前追溯天数,为 None 则使用 STOCK_REPORT_DAYS
|
|
item_type: 过滤类型 None=全部, announcement/research/irm
|
|
"""
|
|
if days_back is None:
|
|
days_back = _report_days()
|
|
|
|
since = date.today() - timedelta(days=days_back)
|
|
since_str = since.strftime("%Y-%m-%d")
|
|
raw_root = Path("data/raw/cninfo")
|
|
if not raw_root.is_dir():
|
|
logger.warning("cninfo raw 目录不存在: {}", raw_root)
|
|
return []
|
|
|
|
items: list[dict] = []
|
|
|
|
for day_dir in sorted(raw_root.glob("*"), reverse=True):
|
|
# 解析目录日期（抓取日期），用于早期跳出循环
|
|
try:
|
|
day_str = day_dir.name
|
|
if len(day_str) != 8:
|
|
continue
|
|
# 目录日期仅用于性能优化：如果目录日期太旧(>days_back*2)，跳过
|
|
day_date = date(int(day_str[:4]), int(day_str[4:6]), int(day_str[6:]))
|
|
if day_date < since - timedelta(days=days_back):
|
|
continue
|
|
except ValueError:
|
|
continue
|
|
|
|
# 从此日期的 index.jsonl 读取
|
|
idx_path = day_dir / "index.jsonl"
|
|
if not idx_path.is_file():
|
|
# 直接读取 JSON 文件
|
|
for jf in sorted(day_dir.glob("*.json"), reverse=True):
|
|
item = _try_load_cninfo_item(jf, code, item_type)
|
|
if item:
|
|
pt = (item.get("publish_time") or "").strip()
|
|
if pt and pt >= since_str:
|
|
items.append(item)
|
|
elif not pt:
|
|
# publish_time 为空（如 irm），仍纳入但标记
|
|
items.append(item)
|
|
else:
|
|
with idx_path.open("r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
rec = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
|
|
# 按股票和类型过滤
|
|
if rec.get("stock_code") != code:
|
|
continue
|
|
if item_type and rec.get("item_type") != item_type:
|
|
continue
|
|
|
|
pt = (rec.get("publish_time") or "").strip()
|
|
# 按 publish_time 过滤（而非抓取日期）
|
|
if pt and pt < since_str:
|
|
continue
|
|
|
|
items.append({
|
|
"title": (rec.get("title") or "").strip(),
|
|
"url": (rec.get("url") or "").strip(),
|
|
"source": "巨潮资讯网",
|
|
"score": 1.0,
|
|
"publish_time": pt,
|
|
"item_type": (rec.get("item_type") or "").strip(),
|
|
"event": rec.get("extra", {}),
|
|
})
|
|
|
|
# 限制同一天/同一类型最多取 50 条
|
|
if len(items) >= 50:
|
|
break
|
|
|
|
return items
|
|
|
|
|
|
def _try_load_cninfo_item(json_path: Path, code: str,
|
|
item_type: str | None) -> dict | None:
|
|
"""从单个 CninfoItem JSON 文件加载(无 index.jsonl 时的回退)。"""
|
|
try:
|
|
data = json.loads(json_path.read_text(encoding="utf-8"))
|
|
except (json.JSONDecodeError, OSError):
|
|
return None
|
|
if data.get("stock_code") != code:
|
|
return None
|
|
if item_type and data.get("item_type") != item_type:
|
|
return None
|
|
return {
|
|
"title": (data.get("title") or "").strip(),
|
|
"url": (data.get("url") or "").strip(),
|
|
"source": "巨潮资讯网",
|
|
"score": 1.0,
|
|
"publish_time": (data.get("publish_time") or "").strip(),
|
|
"item_type": (data.get("item_type") or "").strip(),
|
|
"event": data.get("extra", {}),
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Qdrant 新闻搜索
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def _search_news_from_qdrant(
|
|
emb: Any, store: Any, query_text: str,
|
|
stock_codes: list[str], top_k: int = 30,
|
|
days_back: int | None = None, company_name: str = "",
|
|
) -> list[dict]:
|
|
"""多策略搜索 Qdrant: 股票代码 → 公司名 → 语义。"""
|
|
from vectorstore import SearchFilter
|
|
|
|
if days_back is None:
|
|
days_back = _report_days()
|
|
|
|
vec = emb.embed_one(query_text)
|
|
since = (date.today() - timedelta(days=days_back)).strftime("%Y-%m-%d")
|
|
|
|
# 补全后缀
|
|
codes_with_suffix = []
|
|
for c in stock_codes:
|
|
codes_with_suffix.extend([f"{c}.SZ", f"{c}.SH", f"{c}.BJ", c])
|
|
|
|
hits: list = []
|
|
hits = store.query(query_vector=vec, top_k=top_k,
|
|
filter=SearchFilter(stock_codes=codes_with_suffix,
|
|
publish_date_from=since))
|
|
if not hits and company_name:
|
|
hits = store.query(query_vector=vec, top_k=top_k,
|
|
filter=SearchFilter(company_names=[company_name],
|
|
publish_date_from=since))
|
|
if not hits:
|
|
hits = store.query(query_vector=vec, top_k=top_k,
|
|
filter=SearchFilter(publish_date_from=since))
|
|
return [
|
|
{
|
|
"title": h.title, "url": h.url,
|
|
"source": _SOURCE_NAMES.get(h.source_id, h.source_id),
|
|
"score": round(h.score, 4),
|
|
"publish_time": h.publish_time.isoformat() if h.publish_time else None,
|
|
"event": h.event or {},
|
|
}
|
|
for h in hits
|
|
if _SOURCE_NAMES.get(h.source_id, h.source_id) != "巨潮资讯网"
|
|
]
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# LLM AI 要点分析
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def _generate_ai_summary(company_name: str, announcements: list[dict],
|
|
news: list[dict], research: list[dict],
|
|
irm: list[dict]) -> str:
|
|
"""LLM 生成个股要点分析。"""
|
|
from llm.client import load_llm_config, make_sync_client
|
|
|
|
lines = []
|
|
|
|
if announcements:
|
|
lines.append(f"## 公告 ({len(announcements)} 条)")
|
|
for a in announcements[:10]:
|
|
lines.append(f"- {a['title']} ({a.get('publish_time', '')})")
|
|
|
|
if research:
|
|
lines.append(f"## 调研 ({len(research)} 条)")
|
|
for r in research[:5]:
|
|
lines.append(f"- {r['title']} ({r.get('publish_time', '')})")
|
|
|
|
if news:
|
|
lines.append(f"## 新闻 ({len(news)} 条)")
|
|
for n in news[:10]:
|
|
ev = n.get("event", {})
|
|
summary = ev.get("summary", "")
|
|
lines.append(
|
|
f"- [{n['source']}] {n['title']}"
|
|
+ (f"。{summary}" if summary else "")
|
|
)
|
|
|
|
if irm:
|
|
lines.append(f"## 互动问答 ({len(irm)} 条)")
|
|
for q in irm[:5]:
|
|
lines.append(f"- {q['title']}")
|
|
|
|
if not lines:
|
|
return "暂无足够数据生成 AI 摘要"
|
|
|
|
prompt = f"""以下是 {company_name} 近期的公告、调研、新闻和互动问答:
|
|
|
|
{chr(10).join(lines)[:3500]}
|
|
|
|
请输出 5-8 条要点分析,每条以 "- " 开头:
|
|
1. 最重要的公告或事件是什么?影响如何?
|
|
2. 近期有哪些值得关注的动态?
|
|
3. 市场情绪倾向(利好/利空)?
|
|
4. 后续需要关注什么?
|
|
|
|
直接输出要点列表:"""
|
|
|
|
try:
|
|
config = load_llm_config()
|
|
client = make_sync_client(config)
|
|
resp = client.chat.completions.create(
|
|
model=config.model,
|
|
messages=[{"role": "user", "content": prompt}],
|
|
temperature=0.3, max_tokens=500,
|
|
)
|
|
return (resp.choices[0].message.content or "").strip()
|
|
except Exception as e:
|
|
logger.warning("个股 AI 摘要失败: {}", e)
|
|
return "AI 摘要暂不可用"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# HTML 渲染
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def _clean_markdown(text: str) -> str:
|
|
"""LLM 输出的简单 Markdown 转 HTML 片段。"""
|
|
text = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", text)
|
|
text = re.sub(r"\*(.+?)\*", r"<em>\1</em>", text)
|
|
text = re.sub(r"`(.+?)`", r"<code>\1</code>", text)
|
|
return text
|
|
|
|
|
|
_HTML_TEMPLATE = """<!DOCTYPE html>
|
|
<html lang="zh-CN">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>{company_name}({stock_code}) 个股日报 — {report_date}</title>
|
|
<style>
|
|
:root {{ --bg: #f8f9fa; --card: #fff; --text: #212529; --muted: #6c757d;
|
|
--accent: #2563eb; --border: #dee2e6; --radius: 10px; }}
|
|
* {{ box-sizing: border-box; margin: 0; padding: 0; }}
|
|
body {{ font-family: -apple-system, BlinkMacSystemFont, "PingFang SC", "Microsoft YaHei", sans-serif; background: var(--bg); color: var(--text); line-height: 1.7; padding-bottom: 3rem; }}
|
|
.container {{ max-width: 900px; margin: 0 auto; padding: 1.2rem; }}
|
|
header {{ background: linear-gradient(135deg, #1e293b, #334155); color: #fff; padding: 2rem 0 1.5rem; text-align: center; }}
|
|
header h1 {{ font-size: 1.6rem; }}
|
|
header p {{ color: #94a3b8; margin-top: .3rem; }}
|
|
h2 {{ font-size: 1.2rem; margin: 1.8rem 0 .6rem; padding-bottom: .3rem; border-bottom: 2px solid var(--accent); }}
|
|
.summary {{ background: linear-gradient(135deg, #eff6ff, #f0fdf4); border: 1px solid #93c5fd; border-radius: var(--radius); padding: 1rem 1.2rem; margin: 1rem 0; }}
|
|
.summary li {{ margin: .3rem 0; }}
|
|
.note {{ color: var(--muted); font-size: .85em; margin: .5rem 0; }}
|
|
table {{ width: 100%; border-collapse: collapse; margin: .6rem 0; font-size: .9em; }}
|
|
th, td {{ border: 1px solid var(--border); padding: .4rem .6rem; text-align: left; }}
|
|
th {{ background: #f1f5f9; font-weight: 600; }}
|
|
.badge-pos {{ background: #d1fae5; color: #065f46; padding: .1em .4em; border-radius: 8px; font-size: .78em; }}
|
|
.badge-neg {{ background: #fee2e2; color: #991b1b; padding: .1em .4em; border-radius: 8px; font-size: .78em; }}
|
|
.badge-neu {{ background: #e5e7eb; color: #374151; padding: .1em .4em; border-radius: 8px; font-size: .78em; }}
|
|
a {{ color: var(--accent); text-decoration: none; }}
|
|
a:hover {{ text-decoration: underline; }}
|
|
footer {{ text-align: center; color: var(--muted); font-size: .8em; margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--border); }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<header><div class="container">
|
|
<h1>{company_name} ({stock_code}) 个股日报</h1>
|
|
<p>报告期间: {date_from} ~ {date_to} · 生成于 {generated_at}</p>
|
|
</div></header>
|
|
<main class="container">
|
|
|
|
<h2>一、AI 要点分析</h2>
|
|
<div class="summary">{ai_summary_html}</div>
|
|
|
|
<h2>二、公司公告 ({ann_count} 条)</h2>
|
|
<p class="note">近 {report_days} 日公告,来源 巨潮资讯网</p>
|
|
{ann_table}
|
|
|
|
<h2>三、调研活动 ({research_count} 条)</h2>
|
|
<p class="note">近 {report_days} 日投资者关系活动,来源 巨潮资讯网</p>
|
|
{research_table}
|
|
|
|
<h2>四、相关新闻 ({news_count} 条)</h2>
|
|
<p class="note">近 {report_days} 日财经新闻</p>
|
|
{news_table}
|
|
|
|
<h2>五、互动问答 ({irm_count} 条)</h2>
|
|
<p class="note">近 {report_days} 日互动易平台问答</p>
|
|
{irm_table}
|
|
|
|
</main>
|
|
<footer><div class="container"><p>A 股 Deep Research · 个股日报 · {generated_at}</p></div></footer>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def _render_table(items: list[dict], max_rows: int = 10) -> str:
|
|
if not items:
|
|
return "<p>暂无数据</p>"
|
|
rows = []
|
|
for i, item in enumerate(items[:max_rows], 1):
|
|
ev = item.get("event", {})
|
|
sentiment = ev.get("sentiment", "")
|
|
badge = {"positive": "badge-pos", "negative": "badge-neg"}.get(sentiment, "badge-neu")
|
|
icon = {"positive": "🟢", "negative": "🔴", "neutral": "⚪"}.get(sentiment, "")
|
|
short_title = item["title"][:60]
|
|
if len(item["title"]) > 60:
|
|
short_title += "..."
|
|
date_str = (item.get("publish_time") or "")[:10]
|
|
url = item.get("url", "")
|
|
title_cell = (
|
|
f'<a href="{url}" target="_blank" title="{item["title"]}">{short_title}</a>'
|
|
if url else short_title
|
|
)
|
|
rows.append(
|
|
f'<tr><td>{i}</td>'
|
|
f'<td><span class="{badge}">{icon}</span></td>'
|
|
f'<td>{title_cell}</td>'
|
|
f'<td>{item["source"]}</td>'
|
|
f'<td>{date_str}</td></tr>'
|
|
)
|
|
return (
|
|
f"<table><tr><th>#</th><th></th><th>标题</th><th>来源</th><th>日期</th></tr>"
|
|
f"{''.join(rows)}</table>"
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 单股报告生成
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def _generate_stock_report_with_backend(stock: dict, store: Any, emb: Any, *,
|
|
upload: bool = True) -> Path | None:
|
|
"""为单个股票生成日报(使用共享 Qdrant backend)。"""
|
|
code = stock["code"]
|
|
name = stock["name"]
|
|
days = _report_days()
|
|
|
|
logger.info("生成个股报告: {} ({}) 近{}日", code, name, days)
|
|
|
|
# ---- 从 cninfo v2.0 数据读取 ----
|
|
ann_items = _read_cninfo_items(code, days_back=days, item_type="announcement")
|
|
research_items = _read_cninfo_items(code, days_back=days, item_type="research")
|
|
irm_items = _read_cninfo_items(code, days_back=days, item_type="irm")
|
|
|
|
# ---- 新闻: Qdrant 语义检索 ----
|
|
news_items = _search_news_from_qdrant(
|
|
emb, store, f"{name} {code}", stock_codes=[code],
|
|
top_k=30, days_back=days, company_name=name,
|
|
)
|
|
|
|
# ---- AI 摘要 ----
|
|
ai = _generate_ai_summary(name, ann_items, news_items, research_items, irm_items)
|
|
ai = _clean_markdown(ai)
|
|
ai_html = (
|
|
"<ul>" + "".join(
|
|
f"<li>{ln[2:]}</li>" if ln.startswith("- ") else f"<li>{ln}</li>"
|
|
for ln in ai.strip().splitlines() if ln.strip()
|
|
) + "</ul>"
|
|
if ai else "<p>AI 摘要暂不可用</p>"
|
|
)
|
|
|
|
# ---- 渲染 HTML ----
|
|
today = date.today()
|
|
start_date = today - timedelta(days=days)
|
|
html = _HTML_TEMPLATE.format(
|
|
company_name=name, stock_code=code,
|
|
report_date=today.strftime("%Y-%m-%d"),
|
|
date_from=start_date.strftime("%Y-%m-%d"),
|
|
date_to=today.strftime("%Y-%m-%d"),
|
|
generated_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
|
report_days=days,
|
|
ai_summary_html=ai_html,
|
|
ann_count=len(ann_items),
|
|
ann_table=_render_table(ann_items),
|
|
research_count=len(research_items),
|
|
research_table=_render_table(research_items, max_rows=10),
|
|
news_count=len(news_items),
|
|
news_table=_render_table(news_items),
|
|
irm_count=len(irm_items),
|
|
irm_table=_render_table(irm_items, max_rows=10),
|
|
)
|
|
|
|
# ---- 保存 ----
|
|
out_dir = Path("data/reports/stocks")
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
fname = f"{code}_{name}_个股日报_{today.strftime('%Y%m%d')}.html"
|
|
html_path = out_dir / fname
|
|
html_path.write_text(html, encoding="utf-8")
|
|
logger.info("个股报告已保存: {} ({} KB)", html_path, len(html) // 1024)
|
|
|
|
# ---- 上传 ----
|
|
if upload:
|
|
_upload_stock_report(html_path, today.strftime("%Y%m%d"))
|
|
|
|
return html_path
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 上传
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def _upload_stock_report(html_path: Path, day_str: str) -> bool:
|
|
"""上传个股报告到 Web 服务器。"""
|
|
remote_dir = f"{UPLOAD_BASE}/{day_str}/"
|
|
try:
|
|
r1 = subprocess.run(
|
|
["ssh", UPLOAD_HOST, f"mkdir -p {remote_dir}"],
|
|
timeout=15, capture_output=True, text=True,
|
|
)
|
|
r2 = subprocess.run(
|
|
["scp", str(html_path), f"{UPLOAD_HOST}:{remote_dir}{html_path.name}"],
|
|
timeout=30, capture_output=True, text=True,
|
|
)
|
|
return r1.returncode == 0 and r2.returncode == 0
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 批量生成主入口
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
def generate_all_stock_reports(upload: bool = True) -> int:
|
|
"""为关注列表中所有股票生成个股日报。返回生成的报告数。"""
|
|
watchlist = _load_watchlist()
|
|
if not watchlist:
|
|
logger.warning("关注列表为空,跳过个股报告")
|
|
return 0
|
|
|
|
# 共享 backend(Qdrant + Embedder, 加锁重试)
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
from embedding import make_sync_provider
|
|
from vectorstore import VectorStore, make_qdrant_client
|
|
|
|
emb = make_sync_provider()
|
|
for retry in range(5):
|
|
try:
|
|
client = make_qdrant_client()
|
|
store = VectorStore(client)
|
|
break
|
|
except RuntimeError:
|
|
if retry < 4:
|
|
logger.warning("Qdrant 被占用,{} 秒后重试...", (retry + 1) * 2)
|
|
_time.sleep((retry + 1) * 2)
|
|
else:
|
|
raise
|
|
|
|
count = 0
|
|
for stock in watchlist:
|
|
code = stock.get("code", "")
|
|
name = stock.get("name", "")
|
|
try:
|
|
path = _generate_stock_report_with_backend(stock, store, emb, upload=upload)
|
|
if path:
|
|
count += 1
|
|
except Exception as e:
|
|
logger.error("个股报告生成失败 {} {}: {}", code, name, e)
|
|
|
|
store.close()
|
|
emb.close()
|
|
logger.info("个股报告完成: {}/{} 家", count, len(watchlist))
|
|
return count
|