simon
ff911cf6f7
Token Plan 迁移 / 配置热加载:
- configs/llm_models.yaml: 各场景切到 Token Plan(deepseek-v4.1-flash / qwen3.6-flash)
- 新增 configs/runtime_env.py: .env 按 (mtime_ns, size) 热加载并同步 os.environ,
统一 env_get 取值;llm / embedding / vectorstore / mcp / pipeline 改用 env_get
- configs/loader.py / scripts/run_scheduler.py 等配套调整
- 新增 tests/test_hot_reload.py
日报 AI 摘要为空修复(2026-09-25):
- 根因: 推理模型的 reasoning token 与正文共用 max_tokens, 预算 1500 被"思考"
占满 -> text_tokens=0 / finish_reason=length, 摘要静默为空且不重试
- daily_report 场景新增 max_tokens(默认 4000, YAML 保存即热生效);
LLMConfig 支持可选 max_tokens; 分块预算 800 -> 2000
- _llm_call 拆出 _call_once, 正文为空时自动加倍预算重试(上限 16000),
用尽才降级返回空串; 网络异常重试语义不变
- docs/user-guide.md 新增 FAQ; continuation.md 记录本次排查
- 已重跑 2026-09-25 日报(report_id=357)补回 466 字摘要
测试: 相关用例 56 passed(test_hot_reload 12 passed);
ruff 无新增问题; 3 个 crawler 既有失败与本改动无关
1199 lines
46 KiB
Python
1199 lines
46 KiB
Python
"""每日摘要报告生成器 v2.0。
|
||
|
||
输出 HTML 日报,包含:
|
||
一、AI 摘要 (最新新闻 + {cninfo_days}d 公告/调研/互动)
|
||
二、重要事件: 新闻 (最新抓取, importance≥4, 最多20篇)
|
||
三、重要事件: 公告/互动 ({cninfo_days}d, cninfo, 最多20篇)
|
||
四、数据总览 (重要度/各源/M1-M6/情绪/事件类型分布)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re as _re
|
||
import subprocess
|
||
import time
|
||
from collections import Counter
|
||
from datetime import date, datetime, timedelta
|
||
from pathlib import Path
|
||
from typing import TYPE_CHECKING, Any
|
||
|
||
if TYPE_CHECKING:
|
||
from llm.client import LLMConfig
|
||
|
||
from loguru import logger
|
||
|
||
from configs.runtime_env import env_get
|
||
from report_db.models import EventRow, ReportData # noqa: F401 - 供 _build_report_data 注解使用
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# 配置
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
UPLOAD_HOST = "simon@doorcome.cn"
|
||
UPLOAD_BASE = "/var/www/html/echart/research"
|
||
|
||
NEWS_DAYS_BACK = 1 # 新闻回溯天数
|
||
|
||
_MAX_HIGH_EVENTS = 20
|
||
|
||
# AI 摘要输出预算（token）。
|
||
# 推理模型（deepseek-v4.1-flash 等）的 reasoning token 与正文共用 max_tokens：
|
||
# 预算过小时"思考"会占满配额，正文为空（finish_reason=length、0 字符），
|
||
# 日报就会没有 AI 摘要。默认值需为 reasoning 预留余量。
|
||
DEFAULT_SUMMARY_MAX_TOKENS = 4000 # 单块 / 合并摘要默认预算
|
||
DEFAULT_SUMMARY_CHUNK_MAX_TOKENS = 2000 # 分块摘要默认预算
|
||
MAX_SUMMARY_MAX_TOKENS = 16000 # 正文为空时预算升级上限
|
||
_MAX_BUDGET_ESCALATIONS = 2 # 正文为空时最多升级预算次数
|
||
|
||
|
||
def _env_int(key: str, default: int) -> int:
|
||
"""读取整数环境变量（热加载 .env）；非法值回退默认。"""
|
||
try:
|
||
return int(env_get(key, str(default)) or default)
|
||
except ValueError:
|
||
logger.warning("环境变量 {} 不是整数, 回退默认 {}", key, default)
|
||
return default
|
||
|
||
|
||
def _env_float(key: str, default: float) -> float:
|
||
"""读取浮点环境变量（热加载 .env）；非法值回退默认。"""
|
||
try:
|
||
return float(env_get(key, str(default)) or default)
|
||
except ValueError:
|
||
logger.warning("环境变量 {} 不是数字, 回退默认 {}", key, default)
|
||
return default
|
||
|
||
|
||
def _cninfo_days_back() -> int:
|
||
"""公告/调研回溯天数（与个股日报共用 STOCK_REPORT_DAYS）。"""
|
||
return _env_int("STOCK_REPORT_DAYS", 15)
|
||
|
||
|
||
def _llm_retry_times() -> int:
|
||
return _env_int("LLM_RETRY_TIMES", 3)
|
||
|
||
|
||
def _llm_retry_backoff_sec() -> float:
|
||
return _env_float("LLM_RETRY_BACKOFF_SEC", 2.0)
|
||
|
||
|
||
def _summary_max_tokens(config: LLMConfig | None = None) -> int:
|
||
"""摘要输出预算：场景配置 max_tokens（热更新）> 代码内置默认。"""
|
||
value = getattr(config, "max_tokens", None)
|
||
try:
|
||
return int(value) if value else DEFAULT_SUMMARY_MAX_TOKENS
|
||
except (TypeError, ValueError):
|
||
logger.warning("场景 max_tokens 非法({!r}), 回退默认 {}", value, DEFAULT_SUMMARY_MAX_TOKENS)
|
||
return DEFAULT_SUMMARY_MAX_TOKENS
|
||
|
||
|
||
def _chunk_max_tokens(config: LLMConfig | None = None) -> int:
|
||
"""分块摘要预算：不超过单块/合并预算，避免小块调用过度消耗。"""
|
||
return min(_summary_max_tokens(config), DEFAULT_SUMMARY_CHUNK_MAX_TOKENS)
|
||
|
||
# 日报新闻回溯窗口（小时）：07:00 生成当日日报时覆盖昨日全天至今晨的新闻
|
||
_NEWS_LOOKBACK_HOURS = 30
|
||
|
||
|
||
def _load_source_names() -> dict[str, str]:
|
||
import yaml
|
||
try:
|
||
with open("configs/sources.yaml", encoding="utf-8") as f:
|
||
data = yaml.safe_load(f)
|
||
return {s["id"]: s["name"] for s in (data.get("sources") or []) if s.get("id")}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def _source_name(src_id: str) -> str:
|
||
return _load_source_names().get(src_id, src_id)
|
||
|
||
|
||
def _load_watchlist_codes() -> set[str]:
|
||
import yaml
|
||
try:
|
||
with open("configs/watchlist.yaml", encoding="utf-8") as f:
|
||
data = yaml.safe_load(f) or {}
|
||
return {it["code"] for it in (data.get("watchlist") or []) if it.get("code")}
|
||
except Exception:
|
||
return set()
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# 数据收集
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def _count_jsonl(path: Path) -> int:
|
||
if not path.is_file():
|
||
return 0
|
||
return sum(1 for _ in open(path, encoding="utf-8"))
|
||
|
||
|
||
def _count_json(pattern: str) -> int:
|
||
return len(list(Path().glob(pattern)))
|
||
|
||
|
||
def _load_events_from_dir(day_str: str) -> list[dict]:
|
||
"""从 data/events/{day_str}/ 加载所有事件。"""
|
||
events: list[dict] = []
|
||
ev_dir = Path(f"data/events/{day_str}")
|
||
if not ev_dir.is_dir():
|
||
return events
|
||
for fp in sorted(ev_dir.glob("*.json")):
|
||
try:
|
||
obj = json.loads(fp.read_text(encoding="utf-8"))
|
||
ev = obj.get("event", {})
|
||
events.append({
|
||
"title": obj.get("title", ""),
|
||
"url": obj.get("url", ""),
|
||
"source_id": obj.get("source_id", ""),
|
||
"sources": obj.get("sources") or [obj.get("source_id", "")],
|
||
"publish_time": obj.get("publish_time"),
|
||
"event": ev,
|
||
})
|
||
except (json.JSONDecodeError, OSError):
|
||
pass
|
||
return events
|
||
|
||
|
||
def _collect_news_events(day_str: str) -> dict[str, Any]:
|
||
"""收集新闻事件(排除 cninfo)。
|
||
|
||
读取 `day_str` 与前一天两个事件目录，按 publish_time 过滤最近
|
||
`_NEWS_LOOKBACK_HOURS`（默认 30）小时内的新闻——07:00 生成当日日报时
|
||
可覆盖昨日全天至今晨的新闻。无 publish_time 的事件保留(容错)。
|
||
"""
|
||
day = datetime.strptime(day_str, "%Y%m%d").date()
|
||
prev_day = (day - timedelta(days=1)).strftime("%Y%m%d")
|
||
all_ev = _load_events_from_dir(day_str) + _load_events_from_dir(prev_day)
|
||
|
||
# publish_time 过滤: 最近 30 小时（时间缺失/格式异常的事件保留）
|
||
cutoff = (datetime.now() - timedelta(hours=_NEWS_LOOKBACK_HOURS)).astimezone()
|
||
news_ev: list[dict] = []
|
||
for e in all_ev:
|
||
if e["source_id"] == "cninfo":
|
||
continue
|
||
pt = e.get("publish_time")
|
||
if pt:
|
||
try:
|
||
# naive 时间假定为本地时区, 与带时区(aware)的 cutoff 统一比较
|
||
t = datetime.fromisoformat(pt)
|
||
if t.tzinfo is None:
|
||
t = t.astimezone()
|
||
if t < cutoff:
|
||
continue
|
||
except (ValueError, TypeError):
|
||
pass # 时间格式异常时保留
|
||
news_ev.append(e)
|
||
|
||
sentiments: Counter = Counter()
|
||
importances: Counter = Counter()
|
||
event_types: Counter = Counter()
|
||
for e in news_ev:
|
||
ev = e["event"]
|
||
sentiments[ev.get("sentiment", "?")] += 1
|
||
importances[ev.get("importance", 0)] += 1
|
||
event_types[ev.get("event_type", "?")] += 1
|
||
|
||
# 高重要度: 优先 ≥4, 不足时逐级回退(≥3 → ≥2 → 全部按 importance 排序)
|
||
def _get_high(evs, threshold):
|
||
return sorted(
|
||
[e for e in evs if e["event"].get("importance", 0) >= threshold],
|
||
key=lambda e: -e["event"].get("importance", 0),
|
||
)
|
||
|
||
min_show = 3
|
||
high = _get_high(news_ev, 4)
|
||
hi_threshold = 4
|
||
if len(high) < min_show:
|
||
high = _get_high(news_ev, 3)
|
||
hi_threshold = 3
|
||
if len(high) < min_show:
|
||
high = _get_high(news_ev, 2)
|
||
hi_threshold = 2
|
||
if len(high) < min_show:
|
||
high = sorted(news_ev, key=lambda e: -e["event"].get("importance", 0))
|
||
hi_threshold = 0
|
||
high = high[:_MAX_HIGH_EVENTS]
|
||
|
||
return {
|
||
"total": len(news_ev),
|
||
"high": high,
|
||
"hi_threshold": hi_threshold,
|
||
"sentiments": dict(sentiments),
|
||
"importances": dict(sorted(importances.items())),
|
||
"event_types": dict(event_types.most_common(10)),
|
||
}
|
||
|
||
|
||
def _collect_cninfo_events(today_str: str, days_back: int | None = None) -> dict[str, Any]:
|
||
"""收集近 N 日 cninfo 公告/调研/互动（直接从 processed 数据读取，不依赖 M4 事件抽取）。
|
||
|
||
cninfo 公告/调研数据已结构化（stock_code/name/title/time/type），
|
||
无需经过 LLM 事件抽取即可直接用于日报。
|
||
"""
|
||
if days_back is None:
|
||
days_back = _cninfo_days_back()
|
||
today = datetime.strptime(today_str, "%Y%m%d")
|
||
since_str = (today - timedelta(days=days_back)).strftime("%Y-%m-%d")
|
||
wl_codes = _load_watchlist_codes()
|
||
|
||
items: list[dict] = []
|
||
seen_urls: set[str] = set()
|
||
proc_root = Path("data/processed/cninfo")
|
||
if not proc_root.is_dir():
|
||
return {"total": 0, "high": [], "hi_threshold": 4, "by_day": {},
|
||
"announcement": 0, "research": 0, "irm": 0}
|
||
|
||
for day_dir in sorted(proc_root.glob("*"), reverse=True):
|
||
if not day_dir.is_dir():
|
||
continue
|
||
for fp in sorted(day_dir.glob("*.json"), reverse=True):
|
||
if fp.name == "index.jsonl":
|
||
continue
|
||
try:
|
||
obj = json.loads(fp.read_text(encoding="utf-8"))
|
||
except (json.JSONDecodeError, OSError):
|
||
continue
|
||
|
||
url = obj.get("url") or ""
|
||
if url in seen_urls:
|
||
continue
|
||
seen_urls.add(url)
|
||
|
||
pt = (obj.get("publish_time") or "").strip()
|
||
# 按 publish_time 过滤
|
||
if pt and pt[:10] < since_str:
|
||
continue
|
||
|
||
item_type = obj.get("item_type") or "announcement"
|
||
# 互动易数据跳过（当前无法获取真实数据）
|
||
if item_type == "irm":
|
||
continue
|
||
# 过滤旧数据的 IRM 假阳性（标题为通用占位符或 URL 为 irm 搜索页）
|
||
if "互动问答" in (obj.get("title") or ""):
|
||
continue
|
||
if "irm.cninfo.com.cn" in (obj.get("url") or ""):
|
||
continue
|
||
|
||
title = obj.get("title") or ""
|
||
url = obj.get("url") or ""
|
||
stock_name = obj.get("author") or ""
|
||
content = obj.get("content") or ""
|
||
|
||
# 计算重要度（基于是否在 watchlist 中 + 内容长度）
|
||
code_in_title = ""
|
||
for c in wl_codes:
|
||
if c in title:
|
||
code_in_title = c
|
||
break
|
||
importance = 3 if code_in_title else 2
|
||
if item_type == "research":
|
||
importance = 3 # 调研通常更重要
|
||
|
||
items.append({
|
||
"title": title,
|
||
"url": url,
|
||
"source_id": "cninfo",
|
||
"publish_time": pt,
|
||
"event": {
|
||
"stock_codes": [code_in_title] if code_in_title else [],
|
||
"company_names": [stock_name] if stock_name else [],
|
||
"industries": [],
|
||
"sentiment": "neutral",
|
||
"importance": importance,
|
||
"event_type": {
|
||
"announcement": "公司公告",
|
||
"research": "投资者调研",
|
||
}.get(item_type, "公告"),
|
||
"summary": content[:120] if content else title[:120],
|
||
},
|
||
})
|
||
|
||
# 按发布时间排序（最新在前）
|
||
items.sort(key=lambda e: e.get("publish_time") or "", reverse=True)
|
||
|
||
# 按发布时间的日期分组统计
|
||
by_day: Counter = Counter()
|
||
for e in items:
|
||
pt = (e.get("publish_time") or "")[:10]
|
||
if pt:
|
||
by_day[pt] += 1
|
||
|
||
# 高重要度: 优先 ≥4（调研），逐级回退
|
||
def _get_high(evs, threshold):
|
||
return sorted(
|
||
[e for e in evs if e["event"].get("importance", 0) >= threshold],
|
||
key=lambda e: -e["event"].get("importance", 0),
|
||
)
|
||
min_show = 3
|
||
high = _get_high(items, 4)
|
||
hi_threshold = 4
|
||
if len(high) < min_show:
|
||
high = _get_high(items, 2)
|
||
hi_threshold = 2
|
||
if len(high) < min_show:
|
||
high = sorted(items, key=lambda e: -e["event"].get("importance", 0))
|
||
hi_threshold = 0
|
||
high = high[:_MAX_HIGH_EVENTS]
|
||
|
||
return {
|
||
"total": len(items),
|
||
"high": high,
|
||
"hi_threshold": hi_threshold,
|
||
"by_day": dict(by_day.most_common(7)),
|
||
"announcement": sum(1 for e in items if "公告" in (e["event"].get("event_type", "") or "")),
|
||
"research": sum(1 for e in items if "调研" in (e["event"].get("event_type", "") or "")),
|
||
"irm": sum(1 for e in items if "互动" in (e["event"].get("event_type", "") or "")),
|
||
}
|
||
|
||
|
||
def _score_xwlb_importance(title: str, content: str = "") -> int:
|
||
"""新闻联播条目启发式重要度评分 (1-5)。
|
||
|
||
基于标题+正文关键词匹配，优先匹配高等级:
|
||
5: 直接涉及股市/金融/货币政策
|
||
4: 重大经济/产业政策/能源
|
||
3: 领导人活动/外交/区域发展/外资
|
||
2: 一般国内要闻/农业/生态
|
||
1: 文化/体育/社会/国际简讯
|
||
"""
|
||
text = title + content
|
||
L5 = ["降准", "降息", "印花税", "IPO", "注册制", "退市",
|
||
"并购重组", "增持", "回购", "证券", "股市", "上市"]
|
||
L4 = ["经济", "财政", "税收", "国债", "专项债", "碳",
|
||
"产业", "制造业", "新能源", "芯片", "半导体", "人工智能",
|
||
"算力", "平台经济", "房地产", "外贸", "消费", "投资",
|
||
"供应链", "能源", "电力"]
|
||
L3 = ["习近平", "李强", "总理", "主席", "会谈", "访问",
|
||
"自贸区", "长三角", "粤港澳", "一带一路",
|
||
"央企", "国企", "营商环境", "外资", "达沃斯"]
|
||
L2 = ["会议", "改革", "立法", "监管", "粮食", "农业",
|
||
"水利", "铁路", "公路", "港口", "生态", "救灾"]
|
||
|
||
if any(kw in text for kw in L5):
|
||
return 5
|
||
if any(kw in text for kw in L4):
|
||
return 4
|
||
if any(kw in text for kw in L3):
|
||
return 3
|
||
if any(kw in text for kw in L2):
|
||
return 2
|
||
return 1
|
||
|
||
|
||
def _collect_xwlb(day_str: str) -> dict[str, Any]:
|
||
"""收集新闻联播要闻（从 doorcome API /api/xwlbFine/ 获取）。
|
||
|
||
《新闻联播》每天 19:00 播出：日报在早上生成时当日联播尚未播出，
|
||
因此固定取 `day_str` 前一日（最近一期已播出）的联播数据。
|
||
|
||
API 返回 AI 精编后的独立新闻条目（含标题+正文），
|
||
跳过第 1 条"内容提要"（仅为节目开场白）。
|
||
|
||
返回: {"items": [event_dict, ...], "date": "MM月DD日", "source_date": "前一日"}
|
||
"""
|
||
import urllib.request
|
||
|
||
# 取前一晚(已播出)的联播：day_str 前一天
|
||
prev_day = (datetime.strptime(day_str, "%Y%m%d") - timedelta(days=1)).strftime("%Y%m%d")
|
||
result: dict[str, Any] = {"items": [], "date": "", "source_date": prev_day}
|
||
|
||
api_url = f"https://api.doorcome.cn/api/xwlbFine/?start_date={prev_day}&end_date={prev_day}"
|
||
try:
|
||
req = urllib.request.Request(api_url)
|
||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||
body = json.loads(resp.read().decode("utf-8"))
|
||
except Exception as e:
|
||
logger.warning("新闻联播 API 请求失败: {}", e)
|
||
return result
|
||
|
||
raw_news = body.get("data", {}).get("news", [])
|
||
if not raw_news:
|
||
return result
|
||
|
||
# 提取日期
|
||
dates = {n.get("news_days", "") for n in raw_news if n.get("news_days")}
|
||
if dates:
|
||
d = min(dates)
|
||
result["date"] = f"{d[5:7]}月{d[8:10]}日"
|
||
result["source_date"] = prev_day
|
||
|
||
# 转换为事件格式，跳过第 1 条（内容提要/开场白）
|
||
events: list[dict] = []
|
||
for n in raw_news:
|
||
sid = n.get("daily_sub_id", 0)
|
||
if sid <= 1: # 跳过"内容提要"
|
||
continue
|
||
title = n.get("news_title", "")
|
||
content = n.get("news_improve", "")
|
||
importance = _score_xwlb_importance(title, content)
|
||
events.append({
|
||
"title": title[:100],
|
||
"url": "", # 新闻联播无独立文章链接
|
||
"source_id": "xwlb",
|
||
"publish_time": n.get("news_days", ""),
|
||
"event": {
|
||
"stock_codes": [],
|
||
"company_names": [],
|
||
"industries": [],
|
||
"sentiment": "neutral",
|
||
"importance": importance,
|
||
"event_type": "新闻联播",
|
||
"summary": content[:80] if content else title[:80],
|
||
},
|
||
})
|
||
|
||
# 按重要度降序
|
||
events.sort(key=lambda e: (-e["event"]["importance"], e["title"]))
|
||
result["items"] = events
|
||
return result
|
||
|
||
|
||
def _load_article_urls_from_index(index_path: Path) -> list[dict]:
|
||
"""从 index.jsonl 中加载 stage=article 的条目（排除列表页）。"""
|
||
if not index_path.is_file():
|
||
return []
|
||
articles: list[dict] = []
|
||
for line in open(index_path, encoding="utf-8"):
|
||
try:
|
||
obj = json.loads(line)
|
||
if obj.get("stage") == "article" and obj.get("success"):
|
||
articles.append(obj)
|
||
except (json.JSONDecodeError, KeyError):
|
||
pass
|
||
return articles
|
||
|
||
|
||
def _collect_pipeline_stats(day_str: str) -> dict[str, Any]:
|
||
"""收集管道统计数据（仅计文章级条目 + 24h 抓取新鲜度, 用 fetched_at）。"""
|
||
now = datetime.now()
|
||
cutoff_24h = now - timedelta(hours=24)
|
||
|
||
raw_by_source: dict[str, int] = {}
|
||
raw_by_source_24h: dict[str, int] = {}
|
||
raw_total = 0
|
||
raw_total_24h = 0
|
||
|
||
for idx in Path("data/raw").glob(f"*/{day_str}/index.jsonl"):
|
||
src = idx.parent.parent.name
|
||
articles = _load_article_urls_from_index(idx)
|
||
n = len(articles)
|
||
name = _source_name(src)
|
||
raw_by_source[name] = n
|
||
raw_total += n
|
||
|
||
# 统计 24h 内抓取的文章（用 fetched_at；原实现从 URL 猜日期, 对多数源失效导致全 0）
|
||
n_24h = 0
|
||
for art in articles:
|
||
fa = art.get("fetched_at")
|
||
if fa:
|
||
try:
|
||
if datetime.fromisoformat(fa) >= cutoff_24h:
|
||
n_24h += 1
|
||
except ValueError:
|
||
n_24h += 1 # 时间格式异常时保守计入
|
||
else:
|
||
n_24h += 1 # 时间缺失时保守计入
|
||
raw_by_source_24h[name] = n_24h
|
||
raw_total_24h += n_24h
|
||
|
||
# cninfo raw（仅计文章级条目）
|
||
cninfo_raw = 0
|
||
for idx in Path("data/raw/cninfo").glob("*/index.jsonl"):
|
||
cninfo_raw += len(_load_article_urls_from_index(idx))
|
||
|
||
proc = _count_json(f"data/processed/*/{day_str}/*.json")
|
||
deduped = _count_json(f"data/deduped/{day_str}/uniques/*.json")
|
||
dup_path = Path(f"data/deduped/{day_str}/duplicates.jsonl")
|
||
dups = _count_jsonl(dup_path)
|
||
emb_count = _count_json(f"data/embeddings/{day_str}/*.json")
|
||
|
||
qdrant_count = 0
|
||
try:
|
||
from vectorstore import VectorStore, make_qdrant_client
|
||
c = make_qdrant_client()
|
||
s = VectorStore(c)
|
||
qdrant_count = s.count()
|
||
s.close()
|
||
except Exception:
|
||
pass
|
||
|
||
return {
|
||
"raw_total": raw_total,
|
||
"raw_total_24h": raw_total_24h,
|
||
"raw_by_source": raw_by_source,
|
||
"raw_by_source_24h": raw_by_source_24h,
|
||
"cninfo_raw": cninfo_raw,
|
||
"proc": proc,
|
||
"deduped": deduped,
|
||
"dups": dups,
|
||
"emb_count": emb_count,
|
||
"qdrant_count": qdrant_count,
|
||
}
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# AI 摘要
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def _generate_ai_summary(news: dict, cninfo: dict, day_str: str,
|
||
xwlb: dict | None = None) -> str:
|
||
"""LLM 生成 500 字以内日报摘要，囊括全部新闻、公告及新闻联播。"""
|
||
lines: list[str] = []
|
||
|
||
# 新闻联播（如有）
|
||
if xwlb and xwlb.get("items"):
|
||
items = xwlb["items"]
|
||
lines.append(f"## 新闻联播要闻 ({xwlb.get('date', '')}, {len(items)} 条, 按重要度排序)")
|
||
for e in items[:10]:
|
||
ev = e["event"]
|
||
lines.append(f"- [重要度{ev.get('importance', 0)}] {e['title']}")
|
||
|
||
# 新闻（已在 _collect_news_events 中过滤）
|
||
if news["high"]:
|
||
lines.append(f"## 过去 24 小时高重要度新闻 ({len(news['high'])} 条)")
|
||
for e in news["high"][:12]:
|
||
ev = e["event"]
|
||
sentiment = ev.get("sentiment", "")
|
||
s_icon = {"positive": "利好", "negative": "利空", "neutral": "中性"}.get(sentiment, "")
|
||
lines.append(f"- [{s_icon}][{ev.get('event_type', '')}] {e['title']}。{ev.get('summary', '')}")
|
||
|
||
# 公告/调研
|
||
if cninfo["high"]:
|
||
lines.append(f"## 近 {_cninfo_days_back()} 日重要公告/调研 ({len(cninfo['high'])} 条)")
|
||
for e in cninfo["high"][:8]:
|
||
ev = e.get("event", {})
|
||
lines.append(f"- [{ev.get('event_type', '公司公告')}] {e['title']}")
|
||
|
||
if not lines:
|
||
return ""
|
||
|
||
try:
|
||
from llm.client import SCENE_DAILY_REPORT, load_llm_config, make_sync_client
|
||
config = load_llm_config(scene=SCENE_DAILY_REPORT)
|
||
client = make_sync_client(config)
|
||
return _llm_summarize(client, config, lines, day_str)
|
||
except Exception as e:
|
||
logger.warning("AI 摘要生成失败: {}", e)
|
||
return ""
|
||
|
||
|
||
def _split_lines_into_chunks(lines: list[str], max_chars: int = 3000) -> list[list[str]]:
|
||
"""将 lines 按 max_chars 分块，保证每条新闻(line)不被截断。"""
|
||
chunks: list[list[str]] = []
|
||
current: list[str] = []
|
||
current_len = 0
|
||
|
||
for line in lines:
|
||
line_len = len(line) + 1 # +1 for newline
|
||
if current and current_len + line_len > max_chars:
|
||
chunks.append(current)
|
||
current = []
|
||
current_len = 0
|
||
current.append(line)
|
||
current_len += line_len
|
||
|
||
if current:
|
||
chunks.append(current)
|
||
return chunks
|
||
|
||
|
||
def _llm_summarize(client, config: LLMConfig, lines: list[str], day_str: str) -> str:
|
||
"""LLM 摘要：单块直接总结，多块先分段总结再合并。
|
||
|
||
config 为 llm.client.LLMConfig(daily_report 场景),提供 model / temperature。
|
||
"""
|
||
chunks = _split_lines_into_chunks(lines)
|
||
|
||
if len(chunks) == 1:
|
||
return _llm_call(client, config, _build_prompt(chunks[0], day_str))
|
||
|
||
# 多块：每块独立总结
|
||
partials: list[str] = []
|
||
for i, chunk in enumerate(chunks, 1):
|
||
prompt = f"""以下是今日日报素材的第 {i}/{len(chunks)} 部分 (共 {len(lines)} 条, 本批 {len(chunk)} 条)，请用要点总结，每条一行，以 "- " 开头：
|
||
|
||
{chr(10).join(chunk)}
|
||
|
||
直接输出要点列表："""
|
||
result = _llm_call(client, config, prompt, max_tokens=_chunk_max_tokens(config))
|
||
if result:
|
||
partials.append(result)
|
||
logger.info("AI 摘要: 分块 {}/{} 完成 ({} 字)", i, len(chunks), len(result))
|
||
|
||
if not partials:
|
||
logger.warning("AI 摘要: 所有分块均返回空")
|
||
return ""
|
||
|
||
if len(partials) < len(chunks):
|
||
logger.warning("AI 摘要: {}/{} 分块返回空, 仅合并成功部分", len(chunks) - len(partials), len(chunks))
|
||
|
||
# 合并：将各块摘要合成最终日报摘要
|
||
merge_prompt = f"""以下是 {len(partials)} 组分段摘要，请合并为一份完整的日报摘要 ({day_str})：
|
||
|
||
{chr(10).join(f'--- 第{i+1}组 ---{chr(10)}{p}' for i, p in enumerate(partials))}
|
||
|
||
请合并为要点总结，每条一行以 "- " 开头，要求：
|
||
1. 前 3 条为影响最大的事件，说明为什么重要
|
||
2. 汇总近 {_cninfo_days_back()} 日公司公告/调研核心信息
|
||
3. 市场情绪基调（利好/利空/中性）
|
||
4. 值得持续关注的行业或主题
|
||
5. 纯要点，不要开场白/结束语
|
||
6. 总字数 500 字以内
|
||
|
||
直接输出要点列表："""
|
||
return _llm_call(client, config, merge_prompt, max_tokens=_summary_max_tokens(config))
|
||
|
||
|
||
def _build_prompt(lines: list[str], day_str: str) -> str:
|
||
"""构建标准日报摘要 prompt。"""
|
||
return f"""以下是今日需要总结的全部内容（含新闻联播、财经新闻、公司公告），请据此生成日报摘要 ({day_str})：
|
||
|
||
{chr(10).join(lines)}
|
||
|
||
请用要点总结，每条一行，以 "- " 开头，要求：
|
||
1. 前 3 条为过去 24 小时影响最大的事件（优先参考新闻联播中的重大政策信号），说明为什么重要
|
||
2. 汇总近 {_cninfo_days_back()} 日重要公司公告/调研的核心信息
|
||
3. 市场情绪基调（利好/利空/中性）
|
||
4. 值得持续关注的行业或主题
|
||
5. 纯要点，不要开场白/结束语/标题
|
||
6. 总字数控制在 500 字以内
|
||
|
||
直接输出要点列表："""
|
||
|
||
|
||
def _call_once(client, config: LLMConfig, prompt: str, max_tokens: int) -> tuple[str, str | None]:
|
||
"""单次 LLM 调用，返回 (正文, finish_reason)。异常由调用方处理。"""
|
||
resp = client.chat.completions.create(
|
||
model=config.model,
|
||
messages=[
|
||
{"role": "system", "content": "你是 A 股日报撰写助手,输出简洁、有洞察的新闻摘要。"},
|
||
{"role": "user", "content": prompt},
|
||
],
|
||
temperature=config.temperature,
|
||
max_tokens=max_tokens,
|
||
)
|
||
choice = resp.choices[0]
|
||
content = (choice.message.content or "").strip()
|
||
return content, getattr(choice, "finish_reason", None)
|
||
|
||
|
||
def _llm_call(client, config: LLMConfig, prompt: str, max_tokens: int | None = None) -> str:
|
||
"""单次 LLM 调用（带重试 + 空正文预算升级），返回 strip 后的文本。
|
||
|
||
config 为 llm.client.LLMConfig(daily_report 场景),提供 model / temperature /
|
||
max_tokens（未配置 max_tokens 时用 DEFAULT_SUMMARY_MAX_TOKENS）。
|
||
网络等异常按指数退避重试 `LLM_RETRY_TIMES` 次（默认 3），全部失败则抛出最后一次异常。
|
||
若 finish_reason 为 'length' 且正文为空（推理模型 reasoning 占满预算），
|
||
自动加倍预算重试，最多 `_MAX_BUDGET_ESCALATIONS` 次、上限 MAX_SUMMARY_MAX_TOKENS；
|
||
此时返回空字符串而非抛异常，由调用方降级。
|
||
"""
|
||
retry_times = _llm_retry_times()
|
||
backoff_sec = _llm_retry_backoff_sec()
|
||
budget = max_tokens or _summary_max_tokens(config)
|
||
last_exc: Exception | None = None
|
||
escalations = 0
|
||
|
||
for attempt in range(retry_times):
|
||
try:
|
||
content, finish = _call_once(client, config, prompt, budget)
|
||
except Exception as e:
|
||
last_exc = e
|
||
if attempt < retry_times - 1:
|
||
wait = backoff_sec * (2 ** attempt)
|
||
logger.warning(
|
||
"AI 摘要 LLM 调用失败(第 {}/{} 次): {}; {} 秒后重试",
|
||
attempt + 1, retry_times, e, round(wait, 2),
|
||
)
|
||
time.sleep(wait)
|
||
continue
|
||
|
||
if finish == "length":
|
||
logger.warning(
|
||
"AI 摘要可能被截断: max_tokens={} finish_reason=length 实际输出 {} 字符",
|
||
budget, len(content),
|
||
)
|
||
# 正文为空 = 推理占满预算；加倍预算重试（不改动调用方传入的显式预算以外逻辑）
|
||
if not content and escalations < _MAX_BUDGET_ESCALATIONS:
|
||
budget = min(budget * 2, MAX_SUMMARY_MAX_TOKENS)
|
||
escalations += 1
|
||
logger.warning("AI 摘要正文为空(推理占满预算), 提升 max_tokens 至 {} 重试", budget)
|
||
continue
|
||
return content
|
||
|
||
if last_exc is not None:
|
||
logger.error("AI 摘要 LLM 调用重试 {} 次仍失败: {}", retry_times, last_exc)
|
||
raise last_exc
|
||
logger.warning("AI 摘要 {} 次尝试仍未获得正文, 返回空(日报降级为无 AI 摘要)", retry_times)
|
||
return ""
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# HTML 渲染
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
_HTML_TEMPLATE = """<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<title>A 股 Deep Research 日报 — {date}_{time}</title>
|
||
<style>
|
||
:root {{ --bg: #f8f9fa; --card: #fff; --text: #212529; --muted: #6c757d;
|
||
--accent: #2563eb; --border: #dee2e6; --pos: #059669; --neg: #dc2626;
|
||
--neu: #6b7280; --radius: 10px; }}
|
||
* {{ box-sizing: border-box; margin: 0; padding: 0; }}
|
||
body {{ font-family: -apple-system, BlinkMacSystemFont, "PingFang SC", "Microsoft YaHei", sans-serif; background: var(--bg); color: var(--text); line-height: 1.7; padding-bottom: 3rem; }}
|
||
.container {{ max-width: 1000px; margin: 0 auto; padding: 1.2rem; }}
|
||
header {{ background: linear-gradient(135deg, #1e293b, #334155); color: #fff; padding: 2.5rem 0 1.8rem; text-align: center; }}
|
||
header h1 {{ font-size: 1.8rem; }}
|
||
header p {{ color: #94a3b8; margin-top: .4rem; }}
|
||
h2 {{ font-size: 1.3rem; margin: 2rem 0 .8rem; padding-bottom: .4rem; border-bottom: 2px solid var(--accent); }}
|
||
h3 {{ font-size: 1.05rem; margin: 1.2rem 0 .5rem; color: #374151; }}
|
||
|
||
.ai-summary {{ background: linear-gradient(135deg, #eff6ff, #f0fdf4); border: 1px solid #93c5fd; border-radius: var(--radius); padding: 1.2rem 1.5rem; margin: 1rem 0; line-height: 1.9; font-size: .95em; }}
|
||
|
||
.stats-grid {{ display: grid; grid-template-columns: repeat(6, 1fr); gap: .8rem; margin: 1rem 0; }}
|
||
.stat-card {{ background: var(--card); border: 1px solid var(--border); border-radius: var(--radius); padding: 1rem .8rem; text-align: center; }}
|
||
.stat-card .num {{ font-size: 1.6rem; font-weight: 700; }}
|
||
.stat-card .label {{ font-size: .8em; color: var(--muted); margin-top: .2rem; }}
|
||
|
||
.source-grid {{ display: grid; grid-template-columns: repeat(6, 1fr); gap: .6rem; margin: .8rem 0; }}
|
||
.source-item {{ background: var(--card); border: 1px solid var(--border); border-radius: 6px; padding: .6rem; text-align: center; font-size: .85em; }}
|
||
.source-item .s-name {{ color: var(--muted); font-size: .8em; }}
|
||
.source-item .s-count {{ font-size: 1.2rem; font-weight: 600; }}
|
||
|
||
.sentiment-bar {{ display: flex; height: 24px; border-radius: 6px; overflow: hidden; margin: .5rem 0; }}
|
||
.sentiment-bar .pos {{ background: var(--pos); }}
|
||
.sentiment-bar .neg {{ background: var(--neg); }}
|
||
.sentiment-bar .neu {{ background: var(--neu); }}
|
||
.sentiment-legend {{ display: flex; gap: 1rem; margin: .5rem 0; font-size: .9em; }}
|
||
|
||
table {{ width: 100%; border-collapse: collapse; margin: .8rem 0; font-size: .93em; }}
|
||
th, td {{ border: 1px solid var(--border); padding: .5rem .7rem; text-align: left; }}
|
||
th {{ background: #f1f5f9; font-weight: 600; white-space: nowrap; }}
|
||
.event-row:hover {{ background: #f8fafc; }}
|
||
|
||
.badge {{ display: inline-block; padding: .1em .5em; border-radius: 10px; font-size: .78em; font-weight: 600; }}
|
||
.badge-pos {{ background: #d1fae5; color: #065f46; }}
|
||
.badge-neg {{ background: #fee2e2; color: #991b1b; }}
|
||
.badge-neu {{ background: #e5e7eb; color: #374151; }}
|
||
.imp {{ font-weight: 700; }}
|
||
.imp-5 {{ color: #dc2626; }} .imp-4 {{ color: #ea580c; }}
|
||
|
||
.subsection {{ margin: 1rem 0; }}
|
||
|
||
footer {{ text-align: center; color: var(--muted); font-size: .82em; margin-top: 3rem; padding-top: 1.2rem; border-top: 1px solid var(--border); }}
|
||
a {{ color: var(--accent); text-decoration: none; }}
|
||
a:hover {{ text-decoration: underline; }}
|
||
|
||
@media (max-width: 768px) {{
|
||
.stats-grid, .source-grid {{ grid-template-columns: repeat(3, 1fr); }}
|
||
}}
|
||
|
||
.xwlb-list ol {{ padding-left: 1.2em; }}
|
||
.xwlb-list li {{ padding: .35em 0; line-height: 1.55; border-bottom: 1px dashed var(--border); }}
|
||
.xwlb-num {{ color: var(--accent); font-weight: 600; margin-right: .3em; }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<header>
|
||
<div class="container">
|
||
<h1>📊 A 股 Deep Research 日报</h1>
|
||
<p>{date} · 生成于 {generated_at}</p>
|
||
</div>
|
||
</header>
|
||
<main class="container">
|
||
|
||
<!-- ====== 一、AI 摘要 ====== -->
|
||
<h2>一、AI 摘要</h2>
|
||
<div class="ai-summary">{ai_summary}</div>
|
||
|
||
<!-- ====== 二、新闻联播要闻 ====== -->
|
||
{xwlb_section}
|
||
|
||
<!-- ====== 三、重要事件：新闻 ====== -->
|
||
<h2>三、🔥 重要事件：新闻 <small style="color:var(--muted)">({raw_total_24h}/{raw_total} 篇 24h 内, importance ≥ {news_threshold}, 共 {news_high_count} 篇)</small></h2>
|
||
{news_table}
|
||
|
||
<!-- ====== 四、重要事件：公告/互动 ====== -->
|
||
<h2>四、📋 重要事件：公告 / 调研 / 互动 <small style="color:var(--muted)">(近 {cninfo_days} 日, importance ≥ {cninfo_threshold}, 共 {cninfo_high_count} 篇)</small></h2>
|
||
{cninfo_table}
|
||
|
||
<!-- ====== 五、数据总览 ====== -->
|
||
<h2>五、数据总览</h2>
|
||
|
||
<h3>5.1 M1 → M6 管道</h3>
|
||
<div class="stats-grid">
|
||
<div class="stat-card"><div class="num">{raw_total} <small style="font-size:.45em;color:var(--muted)">({raw_total_24h} 24h)</small></div><div class="label">M1 原始文章</div></div>
|
||
<div class="stat-card"><div class="num">{cninfo_raw}</div><div class="label">M1 cninfo</div></div>
|
||
<div class="stat-card"><div class="num">{proc}</div><div class="label">M2 正文提取</div></div>
|
||
<div class="stat-card"><div class="num">{deduped}</div><div class="label">M3 去重唯一</div></div>
|
||
<div class="stat-card"><div class="num">{emb_count}</div><div class="label">M5 向量</div></div>
|
||
<div class="stat-card"><div class="num">{qdrant_count}</div><div class="label">M6 Qdrant</div></div>
|
||
</div>
|
||
|
||
<h3>5.2 各源数据 <small style="color:var(--muted)">({raw_total_24h}/{raw_total} 篇 24h 内)</small></h3>
|
||
<div class="source-grid">
|
||
{source_cards}
|
||
</div>
|
||
|
||
<h3>5.3 情绪分布 <small style="color:var(--muted)">(最新抓取)</small></h3>
|
||
{sentiment_section}
|
||
|
||
<h3>5.4 重要度分布</h3>
|
||
<table>
|
||
<tr><th>重要度</th>{importance_headers}</tr>
|
||
<tr><td>数量</td>{importance_counts}</tr>
|
||
</table>
|
||
|
||
<h3>5.5 事件类型分布</h3>
|
||
<table>
|
||
<tr><th>事件类型</th><th>数量</th></tr>
|
||
{event_type_rows}
|
||
</table>
|
||
|
||
</main>
|
||
<footer>
|
||
<div class="container">
|
||
<p>A 股 Deep Research 私有投研平台 · 自动生成于 {generated_at}</p>
|
||
</div>
|
||
</footer>
|
||
</body>
|
||
</html>"""
|
||
|
||
|
||
def _render_event_table(events: list[dict], show_source: bool = True,
|
||
show_summary: bool = True) -> str:
|
||
"""渲染事件表格。"""
|
||
if not events:
|
||
return "<p>暂无符合条件的数据</p>"
|
||
rows: list[str] = []
|
||
wl_codes = _load_watchlist_codes()
|
||
for i, e in enumerate(events, 1):
|
||
ev = e["event"]
|
||
sentiment = ev.get("sentiment", "")
|
||
icon = {"positive": "🟢", "negative": "🔴", "neutral": "⚪"}.get(sentiment, "")
|
||
badge_cls = {"positive": "badge-pos", "negative": "badge-neg"}.get(sentiment, "badge-neu")
|
||
imp = ev.get("importance", 0)
|
||
imp_cls = f"imp-{imp}" if imp >= 4 else ""
|
||
title = e["title"][:70]
|
||
src_ids = e.get("sources") or [e.get("source_id", "")]
|
||
src = " / ".join(_source_name(s) for s in src_ids if s)
|
||
if len(src_ids) > 1:
|
||
src = f'<span title="多源新闻">{src}</span> 📰'
|
||
codes_in_event = {c.strip().split(".")[0] for c in (ev.get("stock_codes") or [])}
|
||
star = "⭐ " if codes_in_event & wl_codes else ""
|
||
code_str = f" <small>({','.join(list(codes_in_event)[:3])})</small>" if codes_in_event else ""
|
||
|
||
url = e.get("url", "")
|
||
title_cell = f'{star}<a href="{url}" target="_blank">{title}</a>{code_str}' if url else f"{star}{title}{code_str}"
|
||
|
||
cols = [
|
||
f"<td>{i}</td>",
|
||
f'<td><span class="badge {badge_cls}">{icon}</span></td>',
|
||
f"<td>{title_cell}</td>",
|
||
]
|
||
if show_source:
|
||
cols.append(f"<td>{src}</td>")
|
||
cols.append(f'<td class="imp {imp_cls}">{imp}</td>')
|
||
cols.append(f"<td>{ev.get('event_type', '')}</td>")
|
||
if show_summary:
|
||
cols.append(f"<td>{(ev.get('summary', '') or '')[:60]}</td>")
|
||
|
||
rows.append(f'<tr class="event-row">{"".join(cols)}</tr>')
|
||
|
||
headers = ["#", "", "标题"]
|
||
if show_source:
|
||
headers.append("源")
|
||
headers += ["重要度", "事件类型"]
|
||
if show_summary:
|
||
headers.append("摘要")
|
||
header_row = "".join(f"<th>{h}</th>" for h in headers)
|
||
return f"<table><tr>{header_row}</tr>{''.join(rows)}</table>"
|
||
|
||
|
||
def _render_source_cards(raw_by_source: dict[str, int],
|
||
raw_by_source_24h: dict[str, int],
|
||
cninfo_raw: int) -> str:
|
||
"""渲染源数据卡片,每行 6 个，显示总量和 24h 新鲜数。"""
|
||
cards: list[str] = []
|
||
# 新闻源
|
||
for name, count in sorted(raw_by_source.items()):
|
||
fresh = raw_by_source_24h.get(name, 0)
|
||
cards.append(
|
||
f'<div class="source-item">'
|
||
f'<div class="s-count">{count} <small style="font-size:.65em;color:var(--muted)">({fresh} 24h)</small></div>'
|
||
f'<div class="s-name">{name}</div>'
|
||
f'</div>'
|
||
)
|
||
# cninfo
|
||
cards.append(
|
||
f'<div class="source-item" style="border-color:var(--accent)">'
|
||
f'<div class="s-count">{cninfo_raw}</div>'
|
||
f'<div class="s-name">📋 cninfo</div>'
|
||
f'</div>'
|
||
)
|
||
return "\n".join(cards)
|
||
|
||
|
||
def _render_xwlb_section(xwlb: dict | None) -> str:
|
||
"""渲染新闻联播要闻 HTML 板块（重要事件格式，按重要度排序）。"""
|
||
if not xwlb or not xwlb.get("items"):
|
||
return ""
|
||
|
||
items = xwlb["items"]
|
||
date_label = xwlb.get("date", "")
|
||
source_date = xwlb.get("source_date", "")
|
||
|
||
table_html = _render_event_table(items, show_source=False, show_summary=False)
|
||
|
||
return f"""<h2>二、📺 新闻联播 <small style="color:var(--muted)">({date_label}, 共 {len(items)} 条, 按重要度排序)</small></h2>
|
||
{table_html}
|
||
<p style="color:var(--muted);font-size:.8em;margin-top:.5em">
|
||
来源: 央视《新闻联播》· 数据取自 doorcome API (xwlbFine) · {source_date}
|
||
</p>"""
|
||
|
||
|
||
def _render_html(news: dict, cninfo: dict, pipeline: dict,
|
||
ai_summary: str, day_str: str,
|
||
xwlb: dict | None = None) -> str:
|
||
"""组装完整 HTML（M10 起废弃：日报已改为结构化入库，此函数不再被调用，保留以便回退）。"""
|
||
|
||
# AI 摘要 → HTML
|
||
summary_html = _re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", ai_summary)
|
||
summary_html = _re.sub(r"\*(.+?)\*", r"<em>\1</em>", summary_html)
|
||
summary_html = _re.sub(r"`(.+?)`", r"<code>\1</code>", summary_html)
|
||
if summary_html.strip():
|
||
lines = summary_html.strip().splitlines()
|
||
if any(ln.strip().startswith("- ") for ln in lines):
|
||
items = []
|
||
for ln in lines:
|
||
s = ln.strip()
|
||
if s.startswith("- "):
|
||
items.append(f"<li>{s[2:]}</li>")
|
||
elif s:
|
||
items.append(f"<li>{s}</li>")
|
||
summary_html = f"<ul style='padding-left:1.5rem;margin:.5rem 0'>{''.join(items)}</ul>"
|
||
else:
|
||
summary_html = summary_html.replace("\n", "<br>")
|
||
else:
|
||
summary_html = "<p>AI 摘要暂不可用</p>"
|
||
|
||
# 新闻表格
|
||
news_table = _render_event_table(news["high"])
|
||
|
||
# cninfo 表格
|
||
cninfo_table = _render_event_table(cninfo["high"], show_source=False, show_summary=False)
|
||
|
||
# 源数据卡片
|
||
source_cards = _render_source_cards(
|
||
pipeline["raw_by_source"],
|
||
pipeline.get("raw_by_source_24h", {}),
|
||
pipeline["cninfo_raw"],
|
||
)
|
||
|
||
# 情绪
|
||
s = news["sentiments"]
|
||
pos = s.get("positive", 0)
|
||
neg = s.get("negative", 0)
|
||
neu = s.get("neutral", 0)
|
||
total_s = max(pos + neg + neu, 1)
|
||
sentiment_section = (
|
||
f'<div class="sentiment-bar">'
|
||
f'<div class="pos" style="width:{pos/total_s*100:.0f}%"></div>'
|
||
f'<div class="neg" style="width:{neg/total_s*100:.0f}%"></div>'
|
||
f'<div class="neu" style="width:{neu/total_s*100:.0f}%"></div>'
|
||
f'</div>'
|
||
f'<div class="sentiment-legend">'
|
||
f'<span>🟢 利好 {pos} ({pos/total_s:.0%})</span>'
|
||
f'<span>🔴 利空 {neg} ({neg/total_s:.0%})</span>'
|
||
f'<span>⚪ 中性 {neu} ({neu/total_s:.0%})</span>'
|
||
f'</div>'
|
||
)
|
||
|
||
# 重要度
|
||
imps = news["importances"]
|
||
imp_keys = sorted(imps.keys())
|
||
importance_headers = "".join(f"<th>等级 {k}</th>" for k in imp_keys)
|
||
importance_counts = "".join(f"<td>{imps[k]}</td>" for k in imp_keys)
|
||
|
||
# 事件类型
|
||
et = news["event_types"]
|
||
event_type_rows = "\n".join(
|
||
f"<tr><td>{k}</td><td>{v}</td></tr>" for k, v in et.items()
|
||
)
|
||
|
||
return _HTML_TEMPLATE.format(
|
||
date=day_str,
|
||
time=datetime.now().strftime("%H%M"),
|
||
generated_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
ai_summary=summary_html,
|
||
xwlb_section=_render_xwlb_section(xwlb) if xwlb else "",
|
||
news_high_count=len(news["high"]),
|
||
news_threshold=news.get("hi_threshold", 4),
|
||
news_table=news_table,
|
||
cninfo_high_count=len(cninfo["high"]),
|
||
cninfo_threshold=cninfo.get("hi_threshold", 4),
|
||
cninfo_days=_cninfo_days_back(),
|
||
cninfo_table=cninfo_table,
|
||
raw_total=pipeline["raw_total"],
|
||
raw_total_24h=pipeline.get("raw_total_24h", pipeline["raw_total"]),
|
||
cninfo_raw=pipeline["cninfo_raw"],
|
||
proc=pipeline["proc"],
|
||
deduped=pipeline["deduped"],
|
||
emb_count=pipeline["emb_count"],
|
||
qdrant_count=pipeline["qdrant_count"],
|
||
source_cards=source_cards,
|
||
sentiment_section=sentiment_section,
|
||
importance_headers=importance_headers,
|
||
importance_counts=importance_counts,
|
||
event_type_rows=event_type_rows,
|
||
)
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# 生成 + 上传
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def _build_report_data(news: dict, cninfo: dict, pipeline: dict,
|
||
ai_summary: str, day_str: str,
|
||
xwlb: dict | None = None) -> ReportData:
|
||
"""组装结构化日报数据（M10：写入 MySQL 的前置步骤）。
|
||
|
||
事件板块映射：news["high"]→news / cninfo["high"]→cninfo / xwlb["items"]→xwlb。
|
||
数据总览统计以 JSON 快照存入 stats（前端自行解析）。
|
||
"""
|
||
events: list[EventRow] = []
|
||
|
||
def _rows(items: list[dict], section: str) -> None:
|
||
for i, e in enumerate(items, 1):
|
||
ev = e.get("event", {})
|
||
events.append(
|
||
EventRow(
|
||
section=section,
|
||
rank=i,
|
||
importance=ev.get("importance"),
|
||
event_type=ev.get("event_type"),
|
||
title=str(e.get("title", ""))[:512],
|
||
summary=(ev.get("summary") or None),
|
||
sentiment=ev.get("sentiment") or None,
|
||
source=e.get("source_id") or None,
|
||
sources=e.get("sources") or None,
|
||
url=e.get("url") or None,
|
||
)
|
||
)
|
||
|
||
_rows(news.get("high", []), "news")
|
||
_rows(cninfo.get("high", []), "cninfo")
|
||
if xwlb:
|
||
_rows(xwlb.get("items", []), "xwlb")
|
||
|
||
stats: dict[str, Any] = {
|
||
"pipeline": pipeline,
|
||
"news": {
|
||
"total": news.get("total", 0),
|
||
"hi_threshold": news.get("hi_threshold"),
|
||
"sentiments": news.get("sentiments", {}),
|
||
"importances": news.get("importances", {}),
|
||
"event_types": news.get("event_types", {}),
|
||
},
|
||
"cninfo": {
|
||
"total": cninfo.get("total", 0),
|
||
"hi_threshold": cninfo.get("hi_threshold"),
|
||
"by_day": cninfo.get("by_day", {}),
|
||
"announcement": cninfo.get("announcement", 0),
|
||
"research": cninfo.get("research", 0),
|
||
"irm": cninfo.get("irm", 0),
|
||
},
|
||
}
|
||
if xwlb:
|
||
stats["xwlb"] = {"total": len(xwlb.get("items", [])), "date": xwlb.get("date", "")}
|
||
|
||
return ReportData(
|
||
report_date=datetime.strptime(day_str, "%Y%m%d").date(),
|
||
report_type="finance",
|
||
file_name="", # 新生成日报唯一键退化为 (report_date, finance, "")
|
||
generated_at=datetime.now(),
|
||
ai_summary=ai_summary or None,
|
||
stats=stats,
|
||
events=events,
|
||
)
|
||
|
||
|
||
def generate_report(day_str: str | None = None, *, upload: bool = True) -> int | None:
|
||
"""生成每日日报并结构化入库（M10 完全切换，不再生成 HTML）。
|
||
|
||
`upload` 参数保留以兼容 scheduler/pipeline.py 调用，已无实际作用。
|
||
返回 report_id（成功）或 None（无数据/失败）。
|
||
"""
|
||
# 兜底日期按调度时区取(P1-3),与 pipeline 传入的 date_str 语义一致
|
||
if not day_str:
|
||
from .timeutil import today_str
|
||
day_str = today_str()
|
||
logger.info("生成日报: {}", day_str)
|
||
|
||
# 收集数据
|
||
try:
|
||
news = _collect_news_events(day_str)
|
||
cninfo = _collect_cninfo_events(day_str, days_back=_cninfo_days_back())
|
||
pipeline = _collect_pipeline_stats(day_str)
|
||
xwlb = _collect_xwlb(day_str)
|
||
except Exception as e:
|
||
logger.exception("收集日报数据失败: {}", e)
|
||
return None
|
||
|
||
if news["total"] == 0 and cninfo["total"] == 0 and not xwlb.get("items"):
|
||
logger.warning("{} 无数据,跳过日报生成", day_str)
|
||
return None
|
||
|
||
# AI 摘要(新闻联播 + 新闻 + cninfo)
|
||
ai_summary = _generate_ai_summary(news, cninfo, day_str, xwlb=xwlb)
|
||
|
||
# 结构化入库（替代原 HTML 渲染 + 上传）
|
||
report = _build_report_data(news, cninfo, pipeline, ai_summary, day_str, xwlb=xwlb)
|
||
try:
|
||
from report_db import connect, save_report
|
||
|
||
conn = connect()
|
||
try:
|
||
report_id = save_report(conn, report)
|
||
finally:
|
||
conn.close()
|
||
except Exception as e:
|
||
logger.exception("日报入库失败: {}", e)
|
||
return None
|
||
|
||
logger.info("日报已入库: report_id={}", report_id)
|
||
return report_id
|
||
|
||
|
||
def _upload(html_path: Path, file_tag: str) -> bool:
|
||
"""上传 HTML 报告到 Web 服务器（M10 起废弃：不再被调用，保留以便回退）。"""
|
||
today_str = date.today().strftime("%Y%m%d")
|
||
remote_dir = f"{UPLOAD_BASE}/{today_str}/"
|
||
logger.info("上传日报到 {}:{}", UPLOAD_HOST, remote_dir)
|
||
|
||
try:
|
||
r1 = subprocess.run(
|
||
["ssh", UPLOAD_HOST, f"mkdir -p {remote_dir}"],
|
||
timeout=15, capture_output=True, text=True,
|
||
)
|
||
if r1.returncode != 0:
|
||
logger.warning("创建远程目录失败: {}", r1.stderr.strip())
|
||
return False
|
||
r2 = subprocess.run(
|
||
["scp", str(html_path), f"{UPLOAD_HOST}:{remote_dir}finance_news_daily_{file_tag}.html"],
|
||
timeout=30, capture_output=True, text=True,
|
||
)
|
||
if r2.returncode != 0:
|
||
logger.warning("上传日报失败: {}", r2.stderr.strip())
|
||
return False
|
||
logger.info("上传完成: http://doorcome.cn/echart/research/{}/", today_str)
|
||
return True
|
||
except Exception as e:
|
||
logger.warning("上传日报异常(不阻塞): {}", e)
|
||
return False
|