Initial commit
This commit is contained in:
@@ -0,0 +1,25 @@
|
||||
"""定时任务模块 (M7)。
|
||||
|
||||
公共 API:
|
||||
- run_pipeline: 串联执行 M1→M6 全链路
|
||||
- run_step: 执行单个步骤
|
||||
- StepResult / PipelineResult: 结果模型
|
||||
"""
|
||||
|
||||
from .pipeline import (
|
||||
STEP_COMMANDS,
|
||||
STEP_TIMEOUTS,
|
||||
PipelineResult,
|
||||
StepResult,
|
||||
run_pipeline,
|
||||
run_step,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"STEP_COMMANDS",
|
||||
"STEP_TIMEOUTS",
|
||||
"PipelineResult",
|
||||
"StepResult",
|
||||
"run_pipeline",
|
||||
"run_step",
|
||||
]
|
||||
@@ -0,0 +1,217 @@
|
||||
"""定时任务主流程(M7)。
|
||||
|
||||
编排 M1→M6 全链路,每一步调用已有脚本。
|
||||
单步失败记录日志但不阻断后续(后续步骤可能使用旧缓存数据,降级继续)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
|
||||
from loguru import logger
|
||||
|
||||
# 步骤超时(秒)
|
||||
STEP_TIMEOUTS: dict[str, int] = {
|
||||
"crawler": 900, # M1 抓取(含 Playwright 浏览器,13 源约 8-12 min)
|
||||
"xwlb": 60, # M1 新闻联播 API(纯 HTTP,秒级)
|
||||
"extractor": 300, # M2 正文提取
|
||||
"dedup": 120, # M3 去重
|
||||
"llm": 900, # M4 LLM 事件抽取(API 调用,100 篇约 30s 但加限流余量)
|
||||
"embedding": 300, # M5 向量化
|
||||
"qdrant": 300, # M6 入库(数据量大时需较长时间)
|
||||
"report": 30, # 日报生成+上传
|
||||
"cninfo_crawl": 900, # cninfo watchlist URL 驱动(SPA 渲染,每只约 25s)
|
||||
}
|
||||
|
||||
# 步骤对应的 uv run 命令(参数中 {date} 会被替换为实际日期)
|
||||
# crawler 不支持 --date,固定写当天目录; dedup 不加 --reset 以保持增量
|
||||
STEP_COMMANDS: dict[str, list[str]] = {
|
||||
"crawler": ["uv", "run", "python", "-m", "scripts.run_crawler"], # 无 --date
|
||||
"xwlb": ["uv", "run", "python", "-m", "scripts.run_xwlb"],
|
||||
"extractor": ["uv", "run", "python", "-m", "scripts.run_extractor", "--date", "{date}"],
|
||||
"dedup": ["uv", "run", "python", "-m", "scripts.run_dedup", "--date", "{date}"],
|
||||
"llm": ["uv", "run", "python", "-m", "scripts.run_event_extraction", "--date", "{date}"],
|
||||
"embedding": ["uv", "run", "python", "-m", "scripts.run_embedding", "--date", "{date}"],
|
||||
"qdrant": ["uv", "run", "python", "-m", "scripts.run_qdrant_ingest", "--date", "{date}"],
|
||||
"report": [], # 特殊步骤:仅在每天首次定时任务时追加
|
||||
"cninfo_crawl": ["uv", "run", "a-share", "cninfo"],
|
||||
"cninfo_extract": ["uv", "run", "a-share", "extract", "--source", "cninfo", "--date", "{date}"],
|
||||
"cninfo_pdf": ["uv", "run", "a-share", "cninfo", "--enrich-pdf", "--pdf-limit", "100"],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class StepResult:
|
||||
name: str
|
||||
success: bool
|
||||
elapsed_sec: float
|
||||
exit_code: int | None = None
|
||||
tail_msg: str = ""
|
||||
started_at: datetime | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineResult:
|
||||
steps: list[StepResult] = field(default_factory=list)
|
||||
started_at: datetime | None = None
|
||||
finished_at: datetime | None = None
|
||||
|
||||
@property
|
||||
def all_success(self) -> bool:
|
||||
return all(s.success for s in self.steps)
|
||||
|
||||
|
||||
def run_step(name: str, date_str: str) -> StepResult:
|
||||
"""执行单个 pipeline 步骤。
|
||||
|
||||
参数:
|
||||
name: 步骤名(crawler/extractor/.../report)
|
||||
date_str: YYYYMMDD 日期字符串
|
||||
|
||||
返回: StepResult。
|
||||
"""
|
||||
# report 步骤:内部函数,不走子进程
|
||||
# 日报默认统计"昨天"的数据（因为今天的数据由当天的定时任务处理）。
|
||||
# 如果昨天没有事件数据，向前回溯最多 3 天，取最近有数据的日期。
|
||||
if name == "report":
|
||||
started = datetime.now()
|
||||
try:
|
||||
from datetime import timedelta # noqa: E402
|
||||
from pathlib import Path # noqa: E402
|
||||
|
||||
from .reporter import generate_report # noqa: E402
|
||||
|
||||
# 向前回溯找最近有事件数据的日期（最多回溯 3 天）
|
||||
report_date: str | None = None
|
||||
for offset in range(1, 4):
|
||||
candidate = (date.today() - timedelta(days=offset)).strftime("%Y%m%d")
|
||||
ev_dir = Path(f"data/events/{candidate}")
|
||||
if ev_dir.is_dir() and list(ev_dir.glob("*.json")):
|
||||
report_date = candidate
|
||||
break
|
||||
|
||||
if report_date is None:
|
||||
# 没有任何事件数据，仍然尝试生成昨天日报（至少展示管道统计）
|
||||
report_date = (date.today() - timedelta(days=1)).strftime("%Y%m%d")
|
||||
logger.warning(
|
||||
"日报: 近 3 日均无事件数据 ({} ~ {}), 日报将只含管道统计",
|
||||
(date.today() - timedelta(days=3)).strftime("%Y%m%d"),
|
||||
(date.today() - timedelta(days=1)).strftime("%Y%m%d"),
|
||||
)
|
||||
elif report_date != (date.today() - timedelta(days=1)).strftime("%Y%m%d"):
|
||||
logger.warning(
|
||||
"日报: 昨天 ({}) 无事件数据, 回退使用 {}",
|
||||
(date.today() - timedelta(days=1)).strftime("%Y%m%d"),
|
||||
report_date,
|
||||
)
|
||||
|
||||
path = generate_report(report_date, upload=True)
|
||||
elapsed = (datetime.now() - started).total_seconds()
|
||||
ok = path is not None
|
||||
return StepResult(
|
||||
name=name, success=ok, elapsed_sec=elapsed,
|
||||
tail_msg=str(path) if path else f"无数据 (report_date={report_date})",
|
||||
started_at=started,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
elapsed = (datetime.now() - started).total_seconds()
|
||||
logger.exception("日报生成异常: {}", e)
|
||||
return StepResult(name=name, success=False, elapsed_sec=elapsed,
|
||||
tail_msg=str(e)[:200], started_at=started)
|
||||
|
||||
cmd = STEP_COMMANDS.get(name)
|
||||
if cmd is None:
|
||||
return StepResult(name=name, success=False, elapsed_sec=0,
|
||||
tail_msg=f"未知步骤: {name}")
|
||||
|
||||
full_cmd = [arg.replace("{date}", date_str) for arg in cmd]
|
||||
# 超时优先级:
|
||||
# 1. TIMEOUT_{NAME} 环境变量 (单步精确控制)
|
||||
# 2. PIPELINE_STEP_TIMEOUT 环境变量 (全局兜底, 覆盖硬编码)
|
||||
# 3. STEP_TIMEOUTS 硬编码字典 (代码内默认值)
|
||||
# 4. 1800s (最终兜底)
|
||||
import os
|
||||
specific_key = f"TIMEOUT_{name.upper()}"
|
||||
if specific_key in os.environ:
|
||||
timeout = int(os.environ[specific_key])
|
||||
elif "PIPELINE_STEP_TIMEOUT" in os.environ:
|
||||
timeout = int(os.environ["PIPELINE_STEP_TIMEOUT"])
|
||||
else:
|
||||
timeout = STEP_TIMEOUTS.get(name, 1800)
|
||||
started = datetime.now()
|
||||
logger.info("步骤 {} 开始: {}", name, " ".join(full_cmd))
|
||||
|
||||
try:
|
||||
# 不捕获输出,子进程日志直接流到终端(用户能看到每个源的抓取进度)
|
||||
proc = subprocess.run(
|
||||
full_cmd,
|
||||
timeout=timeout,
|
||||
)
|
||||
elapsed = (datetime.now() - started).total_seconds()
|
||||
ok = proc.returncode == 0
|
||||
|
||||
# dedup 返回 1 是"重复率过高"(无新文章的正常场景)
|
||||
if name == "dedup" and proc.returncode == 1:
|
||||
ok = True
|
||||
logger.info("dedup 重复率超过阈值(无新数据场景,视为成功)")
|
||||
|
||||
tail_msg = f"rc={proc.returncode}" if not ok else ""
|
||||
|
||||
if ok:
|
||||
logger.info("步骤 {} 完成 ({}s) ✅", name, elapsed)
|
||||
else:
|
||||
logger.error("步骤 {} 失败 rc={} ({}s)", name, proc.returncode, elapsed)
|
||||
|
||||
return StepResult(
|
||||
name=name, success=ok, elapsed_sec=elapsed,
|
||||
exit_code=proc.returncode, tail_msg=tail_msg,
|
||||
started_at=started,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
elapsed = (datetime.now() - started).total_seconds()
|
||||
logger.error("步骤 {} 超时 (>{:.0f}s)", name, elapsed)
|
||||
return StepResult(name=name, success=False, elapsed_sec=elapsed,
|
||||
tail_msg="超时", started_at=started)
|
||||
except Exception as e: # noqa: BLE001
|
||||
elapsed = (datetime.now() - started).total_seconds()
|
||||
logger.exception("步骤 {} 异常: {}", name, e)
|
||||
return StepResult(name=name, success=False, elapsed_sec=elapsed,
|
||||
tail_msg=str(e)[:200], started_at=started)
|
||||
|
||||
|
||||
def run_pipeline(date_str: str, *, steps: list[str] | None = None) -> PipelineResult:
|
||||
"""串联执行全链路(M1→M6)。
|
||||
|
||||
参数:
|
||||
date_str: YYYYMMDD。
|
||||
steps: 可选步骤列表,默认全部 6 步。
|
||||
"""
|
||||
names = steps or [k for k in STEP_COMMANDS if k not in ("report", "cninfo_crawl", "cninfo_extract", "cninfo_pdf")]
|
||||
result = PipelineResult(started_at=datetime.now())
|
||||
|
||||
for name in names:
|
||||
sr = run_step(name, date_str)
|
||||
result.steps.append(sr)
|
||||
if not sr.success:
|
||||
logger.warning("步骤 {} 失败,后续步骤继续(可能降级)", name)
|
||||
# 步间留一点缓冲
|
||||
time.sleep(0.5)
|
||||
|
||||
result.finished_at = datetime.now()
|
||||
total = (result.finished_at - result.started_at).total_seconds() if result.started_at else 0
|
||||
succ = sum(1 for s in result.steps if s.success)
|
||||
rate = succ / max(len(result.steps), 1)
|
||||
logger.info(
|
||||
"Pipeline 完成: {}/{} 步骤成功 ({:.0%}) 总耗时 {:.0f}s",
|
||||
succ, len(result.steps), rate, total,
|
||||
)
|
||||
|
||||
# 输出摘要
|
||||
for s in result.steps:
|
||||
flag = "✅" if s.success else "❌"
|
||||
logger.info(" {} {} {}s", flag, s.name, s.elapsed_sec)
|
||||
|
||||
return result
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,534 @@
|
||||
"""个股日报生成器 v2.0。
|
||||
|
||||
根据 watchlist.yaml 配置,为每只关注股票生成日报:
|
||||
- AI 要点分析(DeepSeek 生成)
|
||||
- 公告 / 调研 / 互动问答(cninfo v2.0 CninfoItem 数据)
|
||||
- 相关新闻(Qdrant 语义检索)
|
||||
- HTML 报告 + 自动上传
|
||||
|
||||
数据来源:
|
||||
- cninfo 数据: data/raw/cninfo/{YYYYMMDD}/*.json (CninfoItem v2.0 格式)
|
||||
- 新闻: Qdrant 向量检索
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os as _os
|
||||
import re
|
||||
import subprocess
|
||||
import time as _time
|
||||
from datetime import date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
UPLOAD_HOST = "simon@doorcome.cn"
|
||||
UPLOAD_BASE = "/var/www/html/echart/research"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 配置
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _load_source_names() -> dict[str, str]:
|
||||
import yaml
|
||||
try:
|
||||
with open("configs/sources.yaml", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f)
|
||||
names = {s["id"]: s["name"] for s in (data.get("sources") or []) if s.get("id")}
|
||||
except Exception:
|
||||
names = {}
|
||||
names["cninfo"] = "巨潮资讯网"
|
||||
return names
|
||||
|
||||
|
||||
_SOURCE_NAMES = _load_source_names()
|
||||
|
||||
|
||||
def _load_watchlist() -> list[dict]:
|
||||
import yaml
|
||||
try:
|
||||
with open("configs/watchlist.yaml", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
return list(data.get("watchlist") or [])
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _report_days() -> int:
|
||||
"""从 .env 读取报告天数,默认 15。"""
|
||||
return int(_os.environ.get("STOCK_REPORT_DAYS", "15"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# cninfo 数据读取 (v2.0 CninfoItem 格式)
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _read_cninfo_items(
|
||||
code: str,
|
||||
days_back: int | None = None,
|
||||
item_type: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""从 data/raw/cninfo/ 中读取指定股票的 CninfoItem JSON 数据。
|
||||
|
||||
Args:
|
||||
code: 6 位股票代码
|
||||
days_back: 向前追溯天数,为 None 则使用 STOCK_REPORT_DAYS
|
||||
item_type: 过滤类型 None=全部, announcement/research/irm
|
||||
"""
|
||||
if days_back is None:
|
||||
days_back = _report_days()
|
||||
|
||||
since = date.today() - timedelta(days=days_back)
|
||||
since_str = since.strftime("%Y-%m-%d")
|
||||
raw_root = Path("data/raw/cninfo")
|
||||
if not raw_root.is_dir():
|
||||
logger.warning("cninfo raw 目录不存在: {}", raw_root)
|
||||
return []
|
||||
|
||||
items: list[dict] = []
|
||||
|
||||
for day_dir in sorted(raw_root.glob("*"), reverse=True):
|
||||
# 解析目录日期（抓取日期），用于早期跳出循环
|
||||
try:
|
||||
day_str = day_dir.name
|
||||
if len(day_str) != 8:
|
||||
continue
|
||||
# 目录日期仅用于性能优化：如果目录日期太旧(>days_back*2)，跳过
|
||||
day_date = date(int(day_str[:4]), int(day_str[4:6]), int(day_str[6:]))
|
||||
if day_date < since - timedelta(days=days_back):
|
||||
continue
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# 从此日期的 index.jsonl 读取
|
||||
idx_path = day_dir / "index.jsonl"
|
||||
if not idx_path.is_file():
|
||||
# 直接读取 JSON 文件
|
||||
for jf in sorted(day_dir.glob("*.json"), reverse=True):
|
||||
item = _try_load_cninfo_item(jf, code, item_type)
|
||||
if item:
|
||||
pt = (item.get("publish_time") or "").strip()
|
||||
if pt and pt >= since_str:
|
||||
items.append(item)
|
||||
elif not pt:
|
||||
# publish_time 为空（如 irm），仍纳入但标记
|
||||
items.append(item)
|
||||
else:
|
||||
with idx_path.open("r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
# 按股票和类型过滤
|
||||
if rec.get("stock_code") != code:
|
||||
continue
|
||||
if item_type and rec.get("item_type") != item_type:
|
||||
continue
|
||||
|
||||
pt = (rec.get("publish_time") or "").strip()
|
||||
# 按 publish_time 过滤（而非抓取日期）
|
||||
if pt and pt < since_str:
|
||||
continue
|
||||
|
||||
items.append({
|
||||
"title": (rec.get("title") or "").strip(),
|
||||
"url": (rec.get("url") or "").strip(),
|
||||
"source": "巨潮资讯网",
|
||||
"score": 1.0,
|
||||
"publish_time": pt,
|
||||
"item_type": (rec.get("item_type") or "").strip(),
|
||||
"event": rec.get("extra", {}),
|
||||
})
|
||||
|
||||
# 限制同一天/同一类型最多取 50 条
|
||||
if len(items) >= 50:
|
||||
break
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _try_load_cninfo_item(json_path: Path, code: str,
|
||||
item_type: str | None) -> dict | None:
|
||||
"""从单个 CninfoItem JSON 文件加载(无 index.jsonl 时的回退)。"""
|
||||
try:
|
||||
data = json.loads(json_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
if data.get("stock_code") != code:
|
||||
return None
|
||||
if item_type and data.get("item_type") != item_type:
|
||||
return None
|
||||
return {
|
||||
"title": (data.get("title") or "").strip(),
|
||||
"url": (data.get("url") or "").strip(),
|
||||
"source": "巨潮资讯网",
|
||||
"score": 1.0,
|
||||
"publish_time": (data.get("publish_time") or "").strip(),
|
||||
"item_type": (data.get("item_type") or "").strip(),
|
||||
"event": data.get("extra", {}),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Qdrant 新闻搜索
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _search_news_from_qdrant(
|
||||
emb: Any, store: Any, query_text: str,
|
||||
stock_codes: list[str], top_k: int = 30,
|
||||
days_back: int | None = None, company_name: str = "",
|
||||
) -> list[dict]:
|
||||
"""多策略搜索 Qdrant: 股票代码 → 公司名 → 语义。"""
|
||||
from vectorstore import SearchFilter
|
||||
|
||||
if days_back is None:
|
||||
days_back = _report_days()
|
||||
|
||||
vec = emb.embed_one(query_text)
|
||||
since = (date.today() - timedelta(days=days_back)).strftime("%Y-%m-%d")
|
||||
|
||||
# 补全后缀
|
||||
codes_with_suffix = []
|
||||
for c in stock_codes:
|
||||
codes_with_suffix.extend([f"{c}.SZ", f"{c}.SH", f"{c}.BJ", c])
|
||||
|
||||
hits: list = []
|
||||
hits = store.query(query_vector=vec, top_k=top_k,
|
||||
filter=SearchFilter(stock_codes=codes_with_suffix,
|
||||
publish_date_from=since))
|
||||
if not hits and company_name:
|
||||
hits = store.query(query_vector=vec, top_k=top_k,
|
||||
filter=SearchFilter(company_names=[company_name],
|
||||
publish_date_from=since))
|
||||
if not hits:
|
||||
hits = store.query(query_vector=vec, top_k=top_k,
|
||||
filter=SearchFilter(publish_date_from=since))
|
||||
return [
|
||||
{
|
||||
"title": h.title, "url": h.url,
|
||||
"source": _SOURCE_NAMES.get(h.source_id, h.source_id),
|
||||
"score": round(h.score, 4),
|
||||
"publish_time": h.publish_time.isoformat() if h.publish_time else None,
|
||||
"event": h.event or {},
|
||||
}
|
||||
for h in hits
|
||||
if _SOURCE_NAMES.get(h.source_id, h.source_id) != "巨潮资讯网"
|
||||
]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# LLM AI 要点分析
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _generate_ai_summary(company_name: str, announcements: list[dict],
|
||||
news: list[dict], research: list[dict],
|
||||
irm: list[dict]) -> str:
|
||||
"""LLM 生成个股要点分析。"""
|
||||
from llm.client import load_llm_config, make_sync_client
|
||||
|
||||
lines = []
|
||||
|
||||
if announcements:
|
||||
lines.append(f"## 公告 ({len(announcements)} 条)")
|
||||
for a in announcements[:10]:
|
||||
lines.append(f"- {a['title']} ({a.get('publish_time', '')})")
|
||||
|
||||
if research:
|
||||
lines.append(f"## 调研 ({len(research)} 条)")
|
||||
for r in research[:5]:
|
||||
lines.append(f"- {r['title']} ({r.get('publish_time', '')})")
|
||||
|
||||
if news:
|
||||
lines.append(f"## 新闻 ({len(news)} 条)")
|
||||
for n in news[:10]:
|
||||
ev = n.get("event", {})
|
||||
summary = ev.get("summary", "")
|
||||
lines.append(
|
||||
f"- [{n['source']}] {n['title']}"
|
||||
+ (f"。{summary}" if summary else "")
|
||||
)
|
||||
|
||||
if irm:
|
||||
lines.append(f"## 互动问答 ({len(irm)} 条)")
|
||||
for q in irm[:5]:
|
||||
lines.append(f"- {q['title']}")
|
||||
|
||||
if not lines:
|
||||
return "暂无足够数据生成 AI 摘要"
|
||||
|
||||
prompt = f"""以下是 {company_name} 近期的公告、调研、新闻和互动问答:
|
||||
|
||||
{chr(10).join(lines)[:3500]}
|
||||
|
||||
请输出 5-8 条要点分析,每条以 "- " 开头:
|
||||
1. 最重要的公告或事件是什么?影响如何?
|
||||
2. 近期有哪些值得关注的动态?
|
||||
3. 市场情绪倾向(利好/利空)?
|
||||
4. 后续需要关注什么?
|
||||
|
||||
直接输出要点列表:"""
|
||||
|
||||
try:
|
||||
config = load_llm_config()
|
||||
client = make_sync_client(config)
|
||||
resp = client.chat.completions.create(
|
||||
model=config.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.3, max_tokens=500,
|
||||
)
|
||||
return (resp.choices[0].message.content or "").strip()
|
||||
except Exception as e:
|
||||
logger.warning("个股 AI 摘要失败: {}", e)
|
||||
return "AI 摘要暂不可用"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# HTML 渲染
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _clean_markdown(text: str) -> str:
|
||||
"""LLM 输出的简单 Markdown 转 HTML 片段。"""
|
||||
text = re.sub(r"\*\*(.+?)\*\*", r"<strong>\1</strong>", text)
|
||||
text = re.sub(r"\*(.+?)\*", r"<em>\1</em>", text)
|
||||
text = re.sub(r"`(.+?)`", r"<code>\1</code>", text)
|
||||
return text
|
||||
|
||||
|
||||
_HTML_TEMPLATE = """<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{company_name}({stock_code}) 个股日报 — {report_date}</title>
|
||||
<style>
|
||||
:root {{ --bg: #f8f9fa; --card: #fff; --text: #212529; --muted: #6c757d;
|
||||
--accent: #2563eb; --border: #dee2e6; --radius: 10px; }}
|
||||
* {{ box-sizing: border-box; margin: 0; padding: 0; }}
|
||||
body {{ font-family: -apple-system, BlinkMacSystemFont, "PingFang SC", "Microsoft YaHei", sans-serif; background: var(--bg); color: var(--text); line-height: 1.7; padding-bottom: 3rem; }}
|
||||
.container {{ max-width: 900px; margin: 0 auto; padding: 1.2rem; }}
|
||||
header {{ background: linear-gradient(135deg, #1e293b, #334155); color: #fff; padding: 2rem 0 1.5rem; text-align: center; }}
|
||||
header h1 {{ font-size: 1.6rem; }}
|
||||
header p {{ color: #94a3b8; margin-top: .3rem; }}
|
||||
h2 {{ font-size: 1.2rem; margin: 1.8rem 0 .6rem; padding-bottom: .3rem; border-bottom: 2px solid var(--accent); }}
|
||||
.summary {{ background: linear-gradient(135deg, #eff6ff, #f0fdf4); border: 1px solid #93c5fd; border-radius: var(--radius); padding: 1rem 1.2rem; margin: 1rem 0; }}
|
||||
.summary li {{ margin: .3rem 0; }}
|
||||
.note {{ color: var(--muted); font-size: .85em; margin: .5rem 0; }}
|
||||
table {{ width: 100%; border-collapse: collapse; margin: .6rem 0; font-size: .9em; }}
|
||||
th, td {{ border: 1px solid var(--border); padding: .4rem .6rem; text-align: left; }}
|
||||
th {{ background: #f1f5f9; font-weight: 600; }}
|
||||
.badge-pos {{ background: #d1fae5; color: #065f46; padding: .1em .4em; border-radius: 8px; font-size: .78em; }}
|
||||
.badge-neg {{ background: #fee2e2; color: #991b1b; padding: .1em .4em; border-radius: 8px; font-size: .78em; }}
|
||||
.badge-neu {{ background: #e5e7eb; color: #374151; padding: .1em .4em; border-radius: 8px; font-size: .78em; }}
|
||||
a {{ color: var(--accent); text-decoration: none; }}
|
||||
a:hover {{ text-decoration: underline; }}
|
||||
footer {{ text-align: center; color: var(--muted); font-size: .8em; margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--border); }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header><div class="container">
|
||||
<h1>{company_name} ({stock_code}) 个股日报</h1>
|
||||
<p>报告期间: {date_from} ~ {date_to} · 生成于 {generated_at}</p>
|
||||
</div></header>
|
||||
<main class="container">
|
||||
|
||||
<h2>一、AI 要点分析</h2>
|
||||
<div class="summary">{ai_summary_html}</div>
|
||||
|
||||
<h2>二、公司公告 ({ann_count} 条)</h2>
|
||||
<p class="note">近 {report_days} 日公告,来源 巨潮资讯网</p>
|
||||
{ann_table}
|
||||
|
||||
<h2>三、调研活动 ({research_count} 条)</h2>
|
||||
<p class="note">近 {report_days} 日投资者关系活动,来源 巨潮资讯网</p>
|
||||
{research_table}
|
||||
|
||||
<h2>四、相关新闻 ({news_count} 条)</h2>
|
||||
<p class="note">近 {report_days} 日财经新闻</p>
|
||||
{news_table}
|
||||
|
||||
<h2>五、互动问答 ({irm_count} 条)</h2>
|
||||
<p class="note">近 {report_days} 日互动易平台问答</p>
|
||||
{irm_table}
|
||||
|
||||
</main>
|
||||
<footer><div class="container"><p>A 股 Deep Research · 个股日报 · {generated_at}</p></div></footer>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def _render_table(items: list[dict], max_rows: int = 10) -> str:
|
||||
if not items:
|
||||
return "<p>暂无数据</p>"
|
||||
rows = []
|
||||
for i, item in enumerate(items[:max_rows], 1):
|
||||
ev = item.get("event", {})
|
||||
sentiment = ev.get("sentiment", "")
|
||||
badge = {"positive": "badge-pos", "negative": "badge-neg"}.get(sentiment, "badge-neu")
|
||||
icon = {"positive": "🟢", "negative": "🔴", "neutral": "⚪"}.get(sentiment, "")
|
||||
short_title = item["title"][:60]
|
||||
if len(item["title"]) > 60:
|
||||
short_title += "..."
|
||||
date_str = (item.get("publish_time") or "")[:10]
|
||||
url = item.get("url", "")
|
||||
title_cell = (
|
||||
f'<a href="{url}" target="_blank" title="{item["title"]}">{short_title}</a>'
|
||||
if url else short_title
|
||||
)
|
||||
rows.append(
|
||||
f'<tr><td>{i}</td>'
|
||||
f'<td><span class="{badge}">{icon}</span></td>'
|
||||
f'<td>{title_cell}</td>'
|
||||
f'<td>{item["source"]}</td>'
|
||||
f'<td>{date_str}</td></tr>'
|
||||
)
|
||||
return (
|
||||
f"<table><tr><th>#</th><th></th><th>标题</th><th>来源</th><th>日期</th></tr>"
|
||||
f"{''.join(rows)}</table>"
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 单股报告生成
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _generate_stock_report_with_backend(stock: dict, store: Any, emb: Any, *,
|
||||
upload: bool = True) -> Path | None:
|
||||
"""为单个股票生成日报(使用共享 Qdrant backend)。"""
|
||||
code = stock["code"]
|
||||
name = stock["name"]
|
||||
days = _report_days()
|
||||
|
||||
logger.info("生成个股报告: {} ({}) 近{}日", code, name, days)
|
||||
|
||||
# ---- 从 cninfo v2.0 数据读取 ----
|
||||
ann_items = _read_cninfo_items(code, days_back=days, item_type="announcement")
|
||||
research_items = _read_cninfo_items(code, days_back=days, item_type="research")
|
||||
irm_items = _read_cninfo_items(code, days_back=days, item_type="irm")
|
||||
|
||||
# ---- 新闻: Qdrant 语义检索 ----
|
||||
news_items = _search_news_from_qdrant(
|
||||
emb, store, f"{name} {code}", stock_codes=[code],
|
||||
top_k=30, days_back=days, company_name=name,
|
||||
)
|
||||
|
||||
# ---- AI 摘要 ----
|
||||
ai = _generate_ai_summary(name, ann_items, news_items, research_items, irm_items)
|
||||
ai = _clean_markdown(ai)
|
||||
ai_html = (
|
||||
"<ul>" + "".join(
|
||||
f"<li>{ln[2:]}</li>" if ln.startswith("- ") else f"<li>{ln}</li>"
|
||||
for ln in ai.strip().splitlines() if ln.strip()
|
||||
) + "</ul>"
|
||||
if ai else "<p>AI 摘要暂不可用</p>"
|
||||
)
|
||||
|
||||
# ---- 渲染 HTML ----
|
||||
today = date.today()
|
||||
start_date = today - timedelta(days=days)
|
||||
html = _HTML_TEMPLATE.format(
|
||||
company_name=name, stock_code=code,
|
||||
report_date=today.strftime("%Y-%m-%d"),
|
||||
date_from=start_date.strftime("%Y-%m-%d"),
|
||||
date_to=today.strftime("%Y-%m-%d"),
|
||||
generated_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
report_days=days,
|
||||
ai_summary_html=ai_html,
|
||||
ann_count=len(ann_items),
|
||||
ann_table=_render_table(ann_items),
|
||||
research_count=len(research_items),
|
||||
research_table=_render_table(research_items, max_rows=10),
|
||||
news_count=len(news_items),
|
||||
news_table=_render_table(news_items),
|
||||
irm_count=len(irm_items),
|
||||
irm_table=_render_table(irm_items, max_rows=10),
|
||||
)
|
||||
|
||||
# ---- 保存 ----
|
||||
out_dir = Path("data/reports/stocks")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{code}_{name}_个股日报_{today.strftime('%Y%m%d')}.html"
|
||||
html_path = out_dir / fname
|
||||
html_path.write_text(html, encoding="utf-8")
|
||||
logger.info("个股报告已保存: {} ({} KB)", html_path, len(html) // 1024)
|
||||
|
||||
# ---- 上传 ----
|
||||
if upload:
|
||||
_upload_stock_report(html_path, today.strftime("%Y%m%d"))
|
||||
|
||||
return html_path
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 上传
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def _upload_stock_report(html_path: Path, day_str: str) -> bool:
|
||||
"""上传个股报告到 Web 服务器。"""
|
||||
remote_dir = f"{UPLOAD_BASE}/{day_str}/"
|
||||
try:
|
||||
r1 = subprocess.run(
|
||||
["ssh", UPLOAD_HOST, f"mkdir -p {remote_dir}"],
|
||||
timeout=15, capture_output=True, text=True,
|
||||
)
|
||||
r2 = subprocess.run(
|
||||
["scp", str(html_path), f"{UPLOAD_HOST}:{remote_dir}{html_path.name}"],
|
||||
timeout=30, capture_output=True, text=True,
|
||||
)
|
||||
return r1.returncode == 0 and r2.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 批量生成主入口
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def generate_all_stock_reports(upload: bool = True) -> int:
|
||||
"""为关注列表中所有股票生成个股日报。返回生成的报告数。"""
|
||||
watchlist = _load_watchlist()
|
||||
if not watchlist:
|
||||
logger.warning("关注列表为空,跳过个股报告")
|
||||
return 0
|
||||
|
||||
# 共享 backend(Qdrant + Embedder, 加锁重试)
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
from embedding import make_sync_provider
|
||||
from vectorstore import VectorStore, make_qdrant_client
|
||||
|
||||
emb = make_sync_provider()
|
||||
for retry in range(5):
|
||||
try:
|
||||
client = make_qdrant_client()
|
||||
store = VectorStore(client)
|
||||
break
|
||||
except RuntimeError:
|
||||
if retry < 4:
|
||||
logger.warning("Qdrant 被占用,{} 秒后重试...", (retry + 1) * 2)
|
||||
_time.sleep((retry + 1) * 2)
|
||||
else:
|
||||
raise
|
||||
|
||||
count = 0
|
||||
for stock in watchlist:
|
||||
code = stock.get("code", "")
|
||||
name = stock.get("name", "")
|
||||
try:
|
||||
path = _generate_stock_report_with_backend(stock, store, emb, upload=upload)
|
||||
if path:
|
||||
count += 1
|
||||
except Exception as e:
|
||||
logger.error("个股报告生成失败 {} {}: {}", code, name, e)
|
||||
|
||||
store.close()
|
||||
emb.close()
|
||||
logger.info("个股报告完成: {}/{} 家", count, len(watchlist))
|
||||
return count
|
||||
Reference in New Issue
Block a user