diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..29e473f --- /dev/null +++ b/.env.example @@ -0,0 +1,21 @@ +# ========================================== +# English Financial News — 敏感信息 +# 复制为 .env 并填入真实值 +# 仅存放密钥、Token、API 地址 +# ========================================== + +# --- DeepSeek --- +DEEPSEEK_API_KEY=sk-your-deepseek-key +DEEPSEEK_BASE_URL=https://api.deepseek.com + +# --- Qwen / 通义千问 --- +QWEN_API_KEY=sk-your-qwen-key +QWEN_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 + +# --- DashScope / 阿里百炼 --- +DASHSCOPE_API_KEY=sk-your-dashscope-key +DASHSCOPE_BASE_URL=https://dashscope.aliyuncs.com/api/v1/services/embeddings/text-embedding/text-embedding + +# --- Qdrant --- +QDRANT_URL=http://localhost:6333 +QDRANT_API_KEY= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dff8073 --- /dev/null +++ b/.gitignore @@ -0,0 +1,48 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +dist/ +build/ +.venv/ +venv/ + +# Environment +.env +.env.local +.env.production + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Project data +data/raw/ +data/processed/ +data/deduped/ +data/events/ +data/embeddings/ +data/qdrant_storage/ +data/reports/ + +# Logs +logs/*.log + +# Cache +.cache/ +*.cache + +# pytest +.pytest_cache/ +htmlcov/ +.coverage + +# uv +uv.lock diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..008a64e --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,716 @@ +# CLAUDE.md + +# Claude Code 开发约束(English Financial News 项目) + +版本:v1.0 + +最后更新:2026-06-21 + +--- + +# 一、项目定位 + +本项目目标: + +构建面向国际财经新闻的私有化 Deep Research 平台。 + +核心能力包括: + +* 英文财经新闻抓取(Crawl4AI,12 个英文源,部署海外); +* 英文正文提取(trafilatura); +* 全文英译中(LLM DeepSeek); +* 投资事件抽取(LLM); +* 双语向量知识库(Qdrant); +* MCP 服务 + Cherry Studio / Claude Code Agent 深度研究; +* 每日 AI 摘要日报。 + +本项目不是: + +* 自动交易系统; +* 股票预测系统; +* 投资顾问系统。 + +Claude Code 必须始终围绕"研究辅助平台"进行设计。 + +--- + +# 二、Claude Code 总体行为准则 + +Claude Code 必须遵守以下原则: + +## 2.1 分阶段开发 + +禁止一次性完成整个项目。 + +必须: + +* 每次只完成一个 Milestone; +* 等待人工验收; +* 验收通过后再进入下一阶段。 + +禁止: + +* 擅自推进后续阶段; +* 推翻已完成模块。 + +Milestone 顺序严格按照 english-news-plan.md 第九节执行: +M0 → M1 → 同步机制 → M2 → M3 → M4 → M5+M6 → M7+日报 → M8 + +--- + +## 2.2 最小改动原则 + +修改代码时: + +优先局部修改。 + +禁止: + +为了优化而重写整个模块。 + +除非明确要求: + +"允许重构"。 + +否则: + +保持向后兼容。 + +--- + +## 2.3 先理解,再编码 + +开始编码前必须: + +明确: + +* 当前目标; +* 输入; +* 输出; +* 验收标准。 + +如果需求冲突: + +必须先提问。 + +不得自行猜测。 + +--- + +## 2.4 部署拓扑意识 + +本项目采用双服务器架构(详见 english-news-plan.md 第二节): + +| 服务器 | SSH | 部署路径 | 职责 | +|--------|-----|---------|------| +| 海外 | `ssh ecs-user@8.217.19.253` | `/opt/intlgrab` | M1 抓取 → rsync 推送 | +| 国内 | `ssh pi@192.168.1.160` | `/home/pi/intlnews` | M2→M8 全链路 + 日报 | + +部署优先级: + +* 抓取代码优先在海外服务器测试; +* 其余代码在国内服务器测试; +* 海外验证通过后,抓取代码同步到国内做全链路联调。 + +编码时注意: + +* 海外侧只负责抓取,无 LLM/Embedding 依赖; +* 国内侧等待 rsync 同步完成后再启动管道; +* 哨兵文件(`SYNC_SENTINEL`)是同步完整性的关键信号; +* `data/raw/` 目录在两台服务器上路径一致。 + +--- + +# 三、开发环境规范 + +## 3.1 Python 版本 + +统一使用: + +Python 3.11 + +禁止: + +* Python 3.13; +* Python 3.10 以下版本。 + +--- + +## 3.2 依赖管理 + +统一使用: + +uv + +禁止: + +requirements.txt 手工维护。 + +依赖定义: + +pyproject.toml + +安装命令: + +uv sync + +新增依赖: + +uv add 包名 + +开发依赖: + +uv add --dev 包名 + +--- + +## 3.3 虚拟环境 + +统一使用: + +.venv + +禁止: + +使用 Conda。 + +禁止: + +多个虚拟环境混用。 + +--- + +# 四、代码规范 + +## 4.1 类型注解 + +所有新增代码必须包含类型注解。 + +示例: + +def search_news( +keyword: str, +top_k: int +) -> list[dict]: +... + +禁止: + +省略类型。 + +--- + +## 4.2 中文说明 + +要求: + +代码使用英文命名。 + +注释与文档使用中文。 + +例如: + +# 新闻去重处理 + +def deduplicate_articles(): +... + +--- + +## 4.3 函数长度 + +单个函数: + +建议 ≤ 50 行。 + +超过: + +必须拆分。 + +--- + +## 4.4 文件长度 + +单文件: + +建议 ≤ 500 行。 + +超过: + +必须拆分模块。 + +--- + +## 4.5 禁止魔法数字 + +禁止: + +importance = 5 + +应写成: + +MAX_IMPORTANCE = 5 + +--- + +## 4.6 数据模型 + +统一使用 Pydantic 定义数据结构。 + +核心模型(定义见 english-news-plan.md 第四节): + +* `EnArticle` — 新闻文章(含中英文标题、正文、词数) +* `EnExtractedEvent` — 投资事件(事件类型、股票代码、情绪、重要度) + +禁止: + +大量裸 dict 传递。 + +--- + +# 五、日志规范 + +统一使用: + +logging + +禁止: + +print() + +日志级别: + +DEBUG + +INFO + +WARNING + +ERROR + +CRITICAL + +日志格式: + +时间 - 模块 - 级别 +消息 + +示例: + +2026-06-21 06:30:00 - translator - INFO +开始翻译 reuters 源文章 15 篇 + +--- + +# 六、异常处理规范 + +禁止: + +except: +pass + +必须: + +记录日志。 + +抛出明确异常。 + +示例: + +except Exception as e: + logger.exception(e) + raise + +--- + +# 七、测试规范 + +新增功能必须提供测试。 + +优先使用: + +pytest + +测试目录: + +tests/ + +命名: + +test_xxx.py + +测试覆盖: + +核心模块必须覆盖。 + +包括: + +* crawler(M1 抓取) +* extractor(M2 正文提取) +* dedup(M3 去重) +* translator(M4a 翻译) +* llm(M4b 事件抽取) +* embedding(M5 向量生成) +* vectorstore(M6 Qdrant 入库/检索) +* scheduler(M7 定时调度) +* mcp_server(M8 MCP 服务) + +--- + +# 八、配置规范 + +## 8.1 禁止硬编码 + +所有配置项(API Key、URL、路径、阈值、超时、并发数等)一律禁止写在代码中。 + +常量设置必须按实际情况分配到以下位置: + +| 配置类型 | 存放位置 | 示例 | +|---------|---------|------| +| 密钥/Token | `.env` | `DEEPSEEK_API_KEY`、`DASHSCOPE_API_KEY` | +| API 地址 | `.env` | `DEEPSEEK_BASE_URL`、`QWEN_BASE_URL` | +| Qdrant 连接 | `.env` | `QDRANT_URL`、`QDRANT_API_KEY` | +| 所有功能配置 | `configs/system.yaml` | 模型名、部署路径、阈值、超时、调度、日志 | +| 新闻源定义 | `configs/sources.yaml` | 源 ID、URL 模板、JS 渲染开关 | + +**原则**:`.env` 仅存 SSH/API 的密钥和地址。其余全部在 `system.yaml`。 + +代码中只允许引用配置,不允许定义配置值。 + +正确示例: + +```python +import os +LLM_API_KEY = os.environ["DEEPSEEK_API_KEY"] +``` + +```python +from yaml import safe_load +config = safe_load(open("configs/system.yaml")) +max_articles = config["crawler"]["max_articles_per_run"] +``` + +错误示例: + +```python +API_KEY = "sk-xxx" # ❌ 硬编码密钥 +LLM_TIMEOUT = 60 # ❌ 魔法数字硬编码 +SOURCES = [{"id": "reuters", ...}] # ❌ 源配置硬编码 +``` + +## 8.2 配置文件清单 + +* `configs/sources.yaml` — 英文财经源定义(id / name / url_pattern / js_render 等) +* `configs/system.yaml` — 系统级业务参数(去重阈值、LLM 超时、日报限制等) +* `.env` — 密钥、服务地址(不入 Git) +* `.env.example` — 脱敏示例(入 Git) + +## 8.3 环境变量读取规范 + +* 始终使用 `os.environ["KEY"]`(失败时抛出明确异常),不使用默认值硬编码; +* 如需默认值,必须在 `configs/system.yaml` 中定义,代码从配置文件读取; +* 禁止在 `os.getenv("KEY", "hardcoded_default")` 中写死默认值。 + +--- + +# 九、Prompt 管理规范 + +Prompt 必须独立维护。 + +目录: + +prompts/ + +禁止: + +在代码中直接拼接长 Prompt。 + +本项目 Prompt 文件: + +* `translation_and_extraction.md` — 翻译 + 事件抽取(单次 LLM 调用合并输出) +* `daily_report.md` — 日报生成 Prompt(五段式结构) +* `search_agent.md` — MCP Agent 深度研究 Prompt + +--- + +# 十、数据库规范 + +Qdrant 为唯一向量数据库。 + +Collection 名称:`en_finance_news` + +SQLite 用于: + +任务状态; +缓存; +调试。 + +禁止: + +引入多种向量数据库。 + +除非用户明确要求。 + +--- + +# 十一、Docker 规范 + +所有服务必须支持 Docker。 + +必须提供: + +docker-compose.yml + +必须支持: + +docker compose up -d + +启动。 + +注意: + +* 海外侧 docker-compose 仅包含 Crawl4AI + rsync daemon; +* 国内侧 docker-compose 包含 Qdrant + 调度器 + MCP 服务。 + +不得依赖: + +手工安装。 + +--- + +# 十二、Git 提交规范 + +完成每个 Milestone 后: + +更新: + +README.md + +continuation.md + +docs/ + +提交信息格式: + +feat: +新增功能 + +fix: +问题修复 + +refactor: +重构 + +docs: +文档更新 + +test: +测试 + +chore: +杂项 + +示例: + +feat: 完成 M1 英文财经新闻抓取模块(Crawl4AI) + +--- + +# 十三、README 更新规范 + +每次功能完成后: + +README 必须更新: + +包括: + +功能说明; + +部署方法(区分海外/国内); + +配置说明; + +示例命令; + +常见问题。 + +禁止: + +README 长期不维护。 + +--- + +# 十四、continuation.md 维护规范 + +Claude Code 每次结束工作前: + +必须更新 continuation.md。 + +内容包括: + +当前 Milestone; + +完成内容; + +待办事项; + +已知问题; + +技术债务; + +下次建议。 + +用于恢复上下文。 + +--- + +# 十五、性能要求 + +目标: + +初始支持 12 个英文财经新闻源(见 english-news-plan.md)。 + +逐步扩展至 ≥ 50 个源。 + +处理吞吐: + +≥ 100 篇 / 分钟(端到端:抓取 → 翻译 → 入库)。 + +Qdrant 检索: + +Top-K 响应时间 ≤ 2 秒。 + +LLM 翻译: + +单篇平均 ≤ 3 秒(DeepSeek flash 模型)。 + +--- + +# 十六、翻译质量规范 + +LLM 翻译必须: + +* 使用 DeepSeek 为默认 Provider(Qwen 为备选); +* System prompt 约束财经翻译风格:准确、简洁、专业术语一致; +* 保留英文原标题(`title`)和中文翻译标题(`title_zh`); +* 保留英文原文(`content_en`)和中文翻译(`content_zh`); +* 中文译文字数控制在原文 1.0×–1.5× 范围。 + +事件抽取必须: + +* 正确识别涉及的美股代码(如 AAPL、TSLA); +* 情绪判断有据可查(利好/利空/中性); +* 重要度 1-5 分,≥ 4 分进入日报"重要事件"板块。 + +--- + +# 十七、安全规范 + +禁止: + +提交: + +.env + +API Key(DeepSeek / DashScope / Qwen) + +Cookie + +Token + +个人隐私数据 + +SSH 私钥 + +必须: + +提供: + +.env.example + +示例配置(脱敏)。 + +--- + +# 十八、禁止事项 + +Claude Code 禁止: + +1. 未经允许重构已验收模块; +2. 一次性生成整个项目; +3. 擅自修改数据库结构(包括 Qdrant Collection Schema); +4. 删除已有测试; +5. 使用 print 调试; +6. 忽略异常; +7. 跳过验收直接进入下一阶段; +8. 引入未经说明的新技术栈; +9. 将 Prompt 写死在代码中; +10. 编写无法运行的伪代码冒充完成; +11. 将海外侧依赖(LLM/Embedding)引入 M1 抓取模块; +12. 擅自新增英文新闻源而不更新 configs/sources.yaml 和 plan。 + +--- + +# 十九、输出格式要求 + +Claude Code 完成任务时必须输出: + +【任务目标】 + +【完成内容】 + +【修改文件】 + +【运行方法】 + +【测试结果】 + +【存在问题】 + +【下一步建议】 + +不得只输出代码。 + +必须提供可验证说明。 + +--- + +# 二十、最高优先级原则 + +当多个原则冲突时,优先级如下: + +第一优先级: + +代码正确、可运行。 + +第二优先级: + +稳定性与可维护性。 + +第三优先级: + +向后兼容。 + +第四优先级: + +性能优化。 + +第五优先级: + +代码优雅。 + +宁可代码普通,也不要复杂炫技。 + +本项目追求: + +"小步迭代、稳定演进、长期维护"。 + +--- + +# 二十一、参考文档 + +* `english-news-plan.md` — 项目总体计划(权威来源) +* `docs/` — 设计文档 +* 对标项目:`news/`(A 股 Deep Research),架构模式可复用 + +—— CLAUDE.md 结束 —— diff --git a/README.md b/README.md index a5c3634..687f8f2 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,90 @@ -# intl_news +# English Financial News +国际财经新闻抓取与深度研究平台。 + +## 核心能力 + +- **M1** 英文财经新闻抓取(Crawl4AI,12 个源,部署海外) +- **M2** 英文正文提取(trafilatura) +- **M3** 三层去重(URL Hash / 内容 Hash / SimHash 模糊) +- **M4** 全文英译中 + 投资事件抽取(LLM DeepSeek) +- **M5** 向量生成(DashScope text-embedding-v3) +- **M6** 双语向量知识库(Qdrant) +- **M7** 定时调度 + 每日 AI 摘要日报 +- **M8** MCP 服务(Cherry Studio / Claude Code Agent) + +## 部署架构 + +``` +海外服务器 国内服务器 +M1 抓取 ──rsync──▶ M2→M3→M4→M5→M6 + ↓ + 日报 + M7 调度 + ↓ + M8 MCP 服务 +``` + +## 快速开始 + +### 环境要求 + +- Python 3.11 +- uv + +### 安装 + +```bash +# 创建虚拟环境并安装依赖 +uv sync + +# 配置环境变量 +cp .env.example .env +# 编辑 .env 填入真实 API Key +``` + +### CLI 命令 + +``` +uv run en-news --help + +# 子命令 +uv run en-news crawl # M1 抓取 +uv run en-news extract # M2 提取 +uv run en-news dedup # M3 去重 +uv run en-news translate # M4 翻译+事件抽取 +uv run en-news embed # M5 向量生成 +uv run en-news index # M6 Qdrant 入库 +uv run en-news search "..." # M6 语义检索 +uv run en-news report # 日报生成 +uv run en-news pipeline # 一键管道 +uv run en-news mcp-server # M8 MCP 服务 +``` + +## 配置 + +| 文件 | 用途 | +|------|------| +| `configs/sources.yaml` | 英文财经新闻源定义 | +| `configs/system.yaml` | 系统级业务参数 | +| `.env` | 密钥 / 服务地址 | + +## 开发 + +```bash +# 安装开发依赖 +uv sync --dev + +# 运行测试 +uv run pytest + +# 代码检查 +uv run ruff check . +``` + +## 项目计划 + +详见 [english-news-plan.md](english-news-plan.md) + +## 许可证 + +MIT diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/cli.py b/app/cli.py new file mode 100644 index 0000000..6d5797b --- /dev/null +++ b/app/cli.py @@ -0,0 +1,255 @@ +"""English Financial News — CLI 入口""" + +import logging +import sys + +from dotenv import load_dotenv +import typer + +# 加载 .env 环境变量(必须在所有模块导入之前) +load_dotenv() + +app = typer.Typer( + name="en-news", + help="国际财经新闻抓取与深度研究平台", + no_args_is_help=True, +) + +# ── 日志配置 ────────────────────────────────────────── + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + stream=sys.stderr, +) + +# 抑制第三方库噪音日志 +for _lib in ("httpx", "httpcore", "openai", "urllib3"): + logging.getLogger(_lib).setLevel(logging.WARNING) + +logger = logging.getLogger("en-news") + + +@app.command() +def crawl( + source: str | None = typer.Option( + None, "--source", "-s", + help="只抓取指定 source_id(不传则全部)", + ), + profile: str | None = typer.Option( + None, "--profile", "-p", + help="Profile 名称(2g_headless / 8g_headful)", + ), +): + """M1: 抓取英文财经新闻(部署海外)""" + import os + from crawler.orchestrator import run_crawl_sync + + if profile: + os.environ["EN_NEWS_PROFILE"] = profile + logger.info("📋 使用 Profile: %s", profile) + + try: + stats = run_crawl_sync(source_filter=source) + typer.echo(f"\n✅ 完成: {stats.sources_crawled} 源, {stats.total_articles} 篇文章") + if stats.sources_failed: + typer.echo(f"⚠️ {stats.sources_failed} 个源有错误") + except FileNotFoundError as e: + typer.echo(f"❌ {e}", err=True) + raise typer.Exit(code=1) + except Exception as e: + logger.exception("抓取失败") + typer.echo(f"❌ 抓取出错: {e}", err=True) + raise typer.Exit(code=1) + + +@app.command() +def extract( + source: str | None = typer.Option( + None, "--source", "-s", + help="只处理指定 source_id(不传则全部)", + ), +): + """M2: 英文正文提取(trafilatura)""" + from extractor.pipeline import process_all_sources + + try: + stats = process_all_sources(source_filter=source) + typer.echo( + f"\n✅ 提取: {stats['sources_processed']} 源, " + f"{stats['total_articles']} 篇, {stats['elapsed_sec']:.1f}s" + ) + except Exception as e: + logger.exception("正文提取失败") + typer.echo(f"❌ 提取出错: {e}", err=True) + raise typer.Exit(code=1) + + +@app.command() +def dedup(): + """M3: 三层去重""" + from dedup.pipeline import dedup_all_sources + + try: + stats = dedup_all_sources() + typer.echo( + f"\n✅ 去重: {stats['sources_processed']} 源, " + f"唯一 {stats['unique']} / 重复 {stats['duplicate']} / " + f"总计 {stats['total_articles']}, {stats['elapsed_sec']:.1f}s" + ) + except Exception as e: + logger.exception("去重失败") + typer.echo(f"❌ 去重出错: {e}", err=True) + raise typer.Exit(code=1) + + +@app.command() +def translate(): + """M4: 全文翻译 + 投资事件抽取(LLM)""" + from llm.pipeline import translate_all_deduped + + try: + stats = translate_all_deduped() + typer.echo( + f"\n✅ 翻译+事件抽取: {stats['success']}/{stats['total']} 篇, " + f"{stats['elapsed_sec']:.1f}s ({stats['provider']}/{stats['model']})" + ) + except Exception as e: + logger.exception("翻译失败") + typer.echo(f"❌ 翻译出错: {e}", err=True) + raise typer.Exit(code=1) + + +@app.command() +def embed(): + """M5: 向量生成""" + from embedding.pipeline import embed_all_events + + try: + stats = embed_all_events() + typer.echo( + f"\n✅ 向量生成: {stats['success']}/{stats['total']} 篇, " + f"{stats['elapsed_sec']:.1f}s ({stats['provider']}/{stats['model']})" + ) + except Exception as e: + logger.exception("向量生成失败") + typer.echo(f"❌ 向量生成出错: {e}", err=True) + raise typer.Exit(code=1) + + +@app.command() +def index( + recreate: bool = typer.Option( + False, "--recreate", + help="重建 collection(会删除已有数据)", + ), +): + """M6: Qdrant 入库""" + from vectorstore.pipeline import get_collection_info, ingest_all_embeddings + + try: + stats = ingest_all_embeddings(recreate=recreate) + typer.echo( + f"\n✅ 入库: {stats['ingested']}/{stats['total']} 条, " + f"{stats['elapsed_sec']:.1f}s" + ) + info = get_collection_info() + typer.echo(f"📊 Collection: {info['name']} — {info['vectors_count']} 条向量") + except Exception as e: + logger.exception("入库失败") + typer.echo(f"❌ 入库出错: {e}", err=True) + raise typer.Exit(code=1) + + +@app.command() +def search( + query: str = typer.Argument(..., help="中文检索查询"), + top_k: int = typer.Option(10, help="返回结果数"), +): + """M6: Qdrant 语义检索""" + from vectorstore.pipeline import search_news + + try: + results = search_news(query, top_k=top_k) + if not results: + typer.echo("🔍 未找到相关结果") + return + + typer.echo(f"\n🔍 搜索: {query}\n") + for i, r in enumerate(results, 1): + typer.echo(f"{i}. [{r.source_id}] score={r.score:.4f}") + typer.echo(f" 📰 {r.title_zh or r.title}") + if r.events: + events_summary = ", ".join( + f"{ev.get('event_type','')}({ev.get('sentiment','')})" + for ev in r.events[:3] + ) + typer.echo(f" 📌 事件: {events_summary}") + typer.echo(f" 🔗 {r.url}") + typer.echo() + except Exception as e: + logger.exception("搜索失败") + typer.echo(f"❌ 搜索出错: {e}", err=True) + raise typer.Exit(code=1) + + +@app.command() +def report(): + """M7: 日报生成""" + from scheduler.reporter import generate_report + + try: + path = generate_report() + if path: + typer.echo(f"\n✅ 日报已生成: {path}") + else: + typer.echo("⚠️ 无数据,跳过日报生成") + except Exception as e: + logger.exception("日报生成失败") + typer.echo(f"❌ 日报生成出错: {e}", err=True) + raise typer.Exit(code=1) + + +@app.command() +def pipeline( + skip_report: bool = typer.Option( + False, "--skip-report", + help="跳过日报生成", + ), +): + """M7: 一键运行完整管道 M2→M6(+ 可选日报)""" + from crawler.utils import get_news_day + from scheduler.pipeline import run_pipeline + + date_str = get_news_day() + typer.echo(f"🚀 开始全链路管道,日期: {date_str}\n") + + try: + result = run_pipeline(date_str, skip_report=skip_report) + + typer.echo(f"\n{'='*50}") + typer.echo(f"Pipeline 完成: {result.success_count}/{len(result.steps)} 成功") + for s in result.steps: + flag = "✅" if s.success else "❌" + typer.echo(f" {flag} {s.name}: {s.message} ({s.elapsed_sec:.0f}s)") + if result.finished_at and result.started_at: + total = (result.finished_at - result.started_at).total_seconds() + typer.echo(f"\n⏱ 总耗时: {total:.0f}s") + except Exception as e: + logger.exception("管道执行失败") + typer.echo(f"❌ 管道执行出错: {e}", err=True) + raise typer.Exit(code=1) + + +@app.command() +def mcp_server(): + """M8: 启动 MCP 服务(供 Claude Code / Cherry Studio 调用)""" + from mcp_server.server import mcp + + typer.echo("🚀 启动国际财经 Deep Research MCP 服务...") + mcp.run() + + +if __name__ == "__main__": + app() diff --git a/configs/profiles/2g_headless.yaml b/configs/profiles/2g_headless.yaml new file mode 100644 index 0000000..4a1ac32 --- /dev/null +++ b/configs/profiles/2g_headless.yaml @@ -0,0 +1,25 @@ +# ════════════════════════════════════════════════════ +# Profile: 2G 轻量 headless stealth +# ════════════════════════════════════════════════════ +# 适用场景: +# - 海外服务器(3.5GB 总内存,约 1.2GB 可用) +# - 国内服务器低负载补充抓取 +# - 10 个 RSS 优先源 + 2 个 Web 回退源 +# 策略: +# - stealth headless(隐藏 webdriver 特征) +# - 串行逐源抓取,内存超限自动 GC +# - 单源 2 小时超时,单页 45 秒超时 +# 部署:bash scripts/domestic_crawl_2g.sh [source_id] +# ════════════════════════════════════════════════════ + +proxy: + enabled: false # 海外服务器无需代理;国内如需,改为 true + +crawler: + max_memory_mb: 1800 # 内存上限 + source_timeout_sec: 7200 # 单源超时 2h + article_delay_sec: 3.0 # 文章间冷却 + page_timeout_sec: 45 # 单页超时 + viewport_width: 1024 + viewport_height: 768 + headful: false # headless stealth diff --git a/configs/profiles/8g_headful.yaml b/configs/profiles/8g_headful.yaml new file mode 100644 index 0000000..adf5827 --- /dev/null +++ b/configs/profiles/8g_headful.yaml @@ -0,0 +1,32 @@ +# ════════════════════════════════════════════════════ +# Profile: 8G headful + HTTP 代理 → Privoxy → SOCKS5 +# ════════════════════════════════════════════════════ +# 适用场景: +# - 国内服务器(7.9GB 总内存,约 6.1GB 可用) +# - 所有源均使用 headful Playwright 浏览器直接抓取 +# - 通过 Privoxy (HTTP:3128) → ss-local (SOCKS5:1088) 代理访问海外网站 +# 策略: +# - headful 模式 + Xvfb 虚拟显示器(更像真人) +# - HTTP 代理 127.0.0.1:3128 +# - 更高视口、更长超时 +# - 串行逐源(内存充裕但仍需控制并发) +# - 单源 3 小时超时,单页 60 秒超时 +# 前置条件: +# sudo apt install xvfb +# Xvfb :99 -screen 0 1280x1024x24 & +# 部署:bash scripts/domestic_crawl_8g.sh [source_id] +# ════════════════════════════════════════════════════ + +proxy: + enabled: true # 国内服务器通过 HTTP 代理访问海外网站 + url: "http://127.0.0.1:3128" + +crawler: + max_memory_mb: 7500 # 内存充裕 + source_timeout_sec: 10800 # 单源超时 3h(反爬源可能更慢) + article_delay_sec: 4.0 # 文章间冷却更长(更像真人) + page_timeout_sec: 60 # 单页超时 60s + viewport_width: 1280 # 更正常的视口 + viewport_height: 900 + headful: true # 有头浏览器 + Xvfb + xvfb_display: ":99" # Xvfb 虚拟显示器 diff --git a/configs/sources.yaml b/configs/sources.yaml new file mode 100644 index 0000000..4df871a --- /dev/null +++ b/configs/sources.yaml @@ -0,0 +1,141 @@ +# 英文财经新闻源配置 +# 每个源的字段说明见 english-news-plan.md M1 章节 + +settings: + concurrency: 5 + request_delay_sec: 2 + user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + +sources: + - id: "reuters" + name: "Reuters" + enabled: true + homepage: "https://www.reuters.com/business/" + article_url_pattern: "/[^/]+/[^/]+/" + js_render: false + max_articles_per_run: 30 + anti_bot_mode: "stealth" # DataDome 反爬(Web 回退,大概率失败) + rss_url: "https://news.google.com/rss/search?q=site:reuters.com+business&hl=en-US&gl=US&ceid=US:en" # Google News RSS 代理(标题+摘要,绕过 DataDome) + # ⚠️ Google News RSS 链接为 Google 跳转 URL,无法获取 reuters.com 原文 URL + # rss_crawler.py 自动识别 Google News RSS 格式,标题去 " - Reuters" 后缀 + + - id: "cnbc" + name: "CNBC" + enabled: true + homepage: "https://www.cnbc.com/world/" + article_url_pattern: "/\\d{4}/\\d{2}/\\d{2}/" + js_render: true + max_articles_per_run: 25 + rss_url: "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114" # US Top News and Analysis + # 其他可用频道(id 参数切换): + # id=15837362 → U.S. News + # id=15839135 → Earnings + # id=10000116 → Retail + + - id: "marketwatch" + name: "MarketWatch" + enabled: true + homepage: "https://www.marketwatch.com/latest-news" + article_url_pattern: "/story/" + js_render: false + max_articles_per_run: 25 + rss_url: "https://feeds.marketwatch.com/marketwatch/topstories" # RSS 优先,绕过 DataDome + anti_bot_mode: "stealth" # RSS 失败回退(headful 在无头服务器不可用) + # 其他可用频道: + # https://feeds.marketwatch.com/marketwatch/marketpulse → MarketPulse + # https://feeds.marketwatch.com/marketwatch/bulletins → Bulletins + + - id: "ft" + name: "Financial Times" + enabled: true + homepage: "https://www.ft.com/world" + article_url_pattern: "/content/" + js_render: false + max_articles_per_run: 20 + rss_url: "https://www.ft.com/rss/home" # FT International Homepage RSS + # 其他可用频道: + # https://www.ft.com/rss/world + # https://www.ft.com/rss/companies + # https://www.ft.com/rss/markets + # ⚠️ RSS 含摘要+缩略图,但全文仍需 Crawl4AI 抓取(FT 付费墙) + + - id: "yahoo_finance" + name: "Yahoo Finance" + enabled: true + homepage: "https://finance.yahoo.com/news/" + article_url_pattern: "/(news|articles)/" # /news/、/economy/.../articles/、/markets/.../articles/ + js_render: true + max_articles_per_run: 30 + rss_url: "https://news.yahoo.com/rss/finance" # Yahoo Finance RSS + + - id: "investing" + name: "Investing.com" + enabled: true + homepage: "https://www.investing.com/news/latest-news" + article_url_pattern: "/news/" + js_render: true + max_articles_per_run: 25 + rss_url: "https://www.investing.com/rss/news_1063.rss" # RSS 绕过反爬 + + - id: "seekingalpha" + name: "Seeking Alpha" + enabled: true + homepage: "https://seekingalpha.com/market-news" + article_url_pattern: "/news/" + js_render: true + max_articles_per_run: 20 + anti_bot_mode: "stealth" # 首页 OK,文章页 PerimeterX 拦截 → stealth + rss_url: "https://seekingalpha.com/feed.xml" # 全站文章 RSS(标题+摘要+链接) + + - id: "barrons" + name: "Barron's" + enabled: true + homepage: "https://www.barrons.com/latest-news" + article_url_pattern: "/articles/" + js_render: false + max_articles_per_run: 20 + anti_bot_mode: "stealth" # HTTP 403 → stealth 回退 + # ⚠️ 无 Barron's 专属原生 RSS(2026-06-21 实测:feeds.content.dowjones.io/rss/barrons→404,feeds.a.dj.com/rss/RSSBarrons.xml→403) + # 替代:使用 Dow Jones RSS World News 作为兜底 + rss_url: "https://feeds.a.dj.com/rss/RSSWorldNews.xml" # Dow Jones World News(非 Barron's 专属) + + - id: "wsj" + name: "Wall Street Journal" + enabled: true + homepage: "https://www.wsj.com/economy" + article_url_pattern: "/articles/" + js_render: false + max_articles_per_run: 20 + anti_bot_mode: "stealth" # DataDome → RSS 优先 + stealth 回退 + rss_url: "https://feeds.a.dj.com/rss/RSSWorldNews.xml" # WSJ World News + # 其他可用频道: + # https://feeds.a.dj.com/rss/RSSMarketsMain.xml → Markets + # https://feeds.a.dj.com/rss/RSSWSJD.xml → WSJ Digital (付费墙内) + + - id: "economist" + name: "The Economist" + enabled: true # ✅ RSS 可用(2026-06-21 验证),RSS 优先;Crawl4AI 回退 + homepage: "https://www.economist.com/finance-and-economics" + article_url_pattern: "/finance-and-economics/\\d{4}/\\d{2}/\\d{2}/" + js_render: true + max_articles_per_run: 15 + anti_bot_mode: "stealth" + rss_url: "https://www.economist.com/finance-and-economics/rss.xml" # Finance & economics RSS(含标题+摘要+链接) + + - id: "forexlive" + name: "ForexLive" + enabled: true + homepage: "https://www.forexlive.com/" + article_url_pattern: "/(news|technical-analysis|Education)/" + js_render: false + max_articles_per_run: 20 + rss_url: "https://www.forexlive.com/feed" # Breaking News RSS(含正文,~91KB/次) + + - id: "zerohedge" + name: "ZeroHedge" + enabled: true + homepage: "https://www.zerohedge.com/" + article_url_pattern: "/(geopolitical|markets|economics|commodities|technology|political|macroeconomics|crypto|energy|health|esg)/" + js_render: true + max_articles_per_run: 15 + rss_url: "http://feeds.feedburner.com/zerohedge/feed" # FeedBurner RSS(含正文,~262KB/次) diff --git a/configs/system.yaml b/configs/system.yaml new file mode 100644 index 0000000..6964e02 --- /dev/null +++ b/configs/system.yaml @@ -0,0 +1,91 @@ +# ════════════════════════════════════════════════════ +# 系统功能配置(非敏感) +# 密钥、API 地址 请在 .env 中配置 +# ════════════════════════════════════════════════════ + +# ── 部署拓扑 ──────────────────────────────────────── +# 海外服务器已弃用(M1-M6 全链路在 domestic 执行) +servers: + overseas_host: "ecs-user@8.217.19.253" + overseas_path: "/opt/intlgrab" + domestic_host: "pi@192.168.1.160" + domestic_path: "/home/pi/intlnews" + +# ── HTTP 代理(Privoxy → ss-local → Shadowsocks 海外访问)── +proxy: + enabled: false # 通过 Profile 8g_headful 启用 + url: "http://127.0.0.1:3128" # HTTP 代理地址 + bypass_domains: [] # 不走代理的域名(如国内 CDN) + +# ── 抓取 ──────────────────────────────────────────── +crawler: + max_memory_mb: 1800 # 内存上限 MB(超限触发 GC) + source_timeout_sec: 7200 # 单源超时秒数(2h) + article_delay_sec: 3.0 # 文章间冷却间隔 + page_timeout_sec: 45 # 单页加载超时 + viewport_width: 1024 + viewport_height: 768 + user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" + headful: false # true=有头浏览器(需 X Server 或 xvfb),false=headless + xvfb_display: ":99" # headful 模式下的虚拟显示器(xvfb-run 自动设置) + +# ── 正文提取 ────────────────────────────────────────── +extractor: + min_content_words: 50 + trafilatura_fallback: true + +# ── 去重 ──────────────────────────────────────────── +dedup: + hamming_distance_threshold: 3 + simhash_window_days: 30 + min_content_length: 100 + +# ── LLM 翻译+事件抽取 ──────────────────────────────── +llm: + provider: "deepseek" + deepseek_model: "deepseek-chat" + qwen_model: "qwen-plus" + timeout_sec: 60 + max_retries: 3 + max_tokens: 8192 + temperature: 0.1 + concurrency: 3 + +# ── Embedding 向量化 ──────────────────────────────── +embedding: + provider: "dashscope" + dashscope_model: "text-embedding-v3" + dimension: 1024 + batch_size: 10 + timeout_sec: 30 + +# ── Qdrant ────────────────────────────────────────── +qdrant: + collection: "en_finance_news" + +# ── 日报 ──────────────────────────────────────────── +report: + max_events: 20 + summary_max_chars: 500 + importance_threshold: 4 + upload_host: "simon@doorcome.cn" + upload_path: "/var/www/html/echart/research" + +# ── 同步 (rsync) ───────────────────────────────────── +# 海外服务器已弃用,以下保留作为参考 +sync: + port: 22 + sentinel: "/tmp/en_news_sync_done" + pack_dir: "/tmp" + +# ── 定时调度 ──────────────────────────────────────── +# 全流程:M1 抓取 → M2→M6 管道 → 日报 +# 每天 06:00 / 12:00 / 18:00 / 22:00 各执行一次 +schedule: + day_cutoff_hour: 6 + times: ["06:00", "12:00", "18:00", "22:00"] + +# ── 日志 ──────────────────────────────────────────── +logging: + level: "INFO" + dir: "logs" diff --git a/continuation.md b/continuation.md new file mode 100644 index 0000000..2f5a251 --- /dev/null +++ b/continuation.md @@ -0,0 +1,146 @@ +# continuation.md — English Financial News 项目状态 + +> 最后更新:2026-07-17 + +--- + +## 2026-07-17 修复 + +### domestic_crawl_8g.sh 无参崩溃 +- `"$1"` 触发 `set -u` → `"${1:-}"` +- 已同步 Pi,语法检查通过 + +--- + +## 当前进度 + +``` +M0 ✅ 项目骨架 +M1 ✅ 新闻抓取(12/12 源,Pi headful Playwright + HTTP 代理) +M2 ✅ 正文提取(trafilatura + MD 回退,增量跳过已处理) +M3 ✅ 三层去重(L1 URL / L2 Content / L3 SimHash,SQLite 指纹库) +M4 ✅ 翻译+事件抽取(DeepSeek v4-flash,事件去重 Prompt 约束) +M5 ✅ 向量生成(DashScope text-embedding-v3,1024 维) +M6 ✅ Qdrant 入库(本地文件模式,语义搜索验证) +M7 ✅ 全链路管道 + 日报(事件级去重 + AI 摘要分批合并 + 防截断) +M8 ✅ MCP 服务(FastMCP 5 个 Tool) +``` + +--- + +## 架构变更(2026-07-14) + +### 之前 + +``` +海外 M1 (crawl) → pack → rsync → Pi M2→M6 → 日报 +``` + +### 现在 + +``` +Pi M1 (headful+proxy) → Pi M2→M6 → 日报 +(海外服务器不再参与流水线) +``` + +### 修改文件 + +| 文件 | 改动 | +|------|------| +| `configs/profiles/8g_headful.yaml` | proxy URL: `socks5://1080` → `http://127.0.0.1:3128` | +| `configs/system.yaml` | 调度时间 `06:00/12:00/18:00/22:00`,同步段标为参考 | +| `scripts/domestic_full.sh` | 重写:移除海外 SSH 触发/sync,2 步:M1 抓取 → 管道+日报 | +| `scripts/domestic_crawl_8g.sh` | 注释更新 | +| `scheduler/jobs.py` | 移除海外 crontab,国内统一 domestic_full.sh 每日 4 次 | + +### 抓取能力测试结果 + +| 源 | 结果 | 说明 | +|----|------|------| +| Forexlive | ✅ 13/13 Web 100% | headful 绕过 RSS 限制,获取正文 | +| MarketWatch | ✅ 3 篇 Web 新增 | DataDome 部分破解 | +| SeekingAlpha | ✅ RSS 稳定 | 30 篇增量跳过,无需 Web | +| WSJ | ✅ RSS 稳定 | Dow Jones RSS 兜底 | +| Barron's | ✅ RSS 稳定 | Dow Jones RSS 兜底 | +| CNBC/Yahoo/ZeroHedge/Economist | ✅ RSS 稳定 | 增量跳过 | +| **Reuters** | ❌ DataDome captcha | headful+proxy+stealth 均无法绕过 | +| **FT** | ❌ 超时 | RSS 超时,首页无可用链接 | +| **Investing.com** | ❌ Cloudflare 拦截 | RSS 403,Web 被 Cloudflare JS 挑战拦截 | + +--- + +## 2026-07-14 会话成果 + +### Pi Shadowsocks 客户端部署 + +- `shadowsocks-libev` + `v2ray-plugin v1.3.2`(arm64)已安装 +- `ss-local` 配置 `0.0.0.0:1088`(SOCKS5),systemd 已启用 +- `privoxy` 配置 `0.0.0.0:3128`(HTTP → SOCKS5),已运行 +- 配置文件 `/etc/shadowsocks-libev/config.json` 已创建(服务端信息待用户填入) + +### 全链路独立化 + +- 海外服务器不再承担 M1 抓取任务 +- Pi 通过 `8g_headful` profile(`EN_NEWS_PROFILE`)执行 headful Playwright + HTTP 代理抓取 +- `domestic_full.sh` 简化为 2 步:M1 抓取 → 管道(含日报) + +--- + +## 服务器状态 + +| 角色 | 地址 | 路径 | 状态 | +|------|------|------|------| +| 海外 | `ecs-user@8.217.19.253` | `/opt/intlgrab` | ⏸️ 不再参与流水线,crontab 待停用 | +| 国内 | `pi@192.168.1.160` | `/home/pi/intlnews` | ✅ 全链路独立运行 | + +--- + +## 定时任务(待设置) + +### Pi crontab(需用户手动执行 `crontab -e`) + +``` +# 全流程:M1 抓取 → M2→M6 管道 → 日报 +0 6 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh >> logs/full.log 2>&1 +0 12 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh >> logs/full.log 2>&1 +0 18 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh >> logs/full.log 2>&1 +0 22 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh >> logs/full.log 2>&1 + +# Xvfb 开机自启 +@reboot Xvfb :99 -screen 0 1280x1024x24 -ac +extension RANDR & +``` + +### 海外服务器 crontab(待停用) + +``` +# 当前仍在运行,需停用 +# 0 6 * * * overseas_crawl.sh +# 0 11 * * * overseas_crawl.sh +# 0 17 * * * overseas_crawl.sh +# 0 23 * * * overseas_crawl.sh +``` + +--- + +## 待办事项 + +1. **用户填入 Shadowsocks 服务端信息** → 启动 ss-local +2. **Pi crontab 设置** → 全流程定时 + Xvfb 开机自启 +3. **海外 crontab 停用** → `crontab -e` 注释掉抓取调度 +4. **Reuters / FT / Investing.com** → 反爬问题未解决,需后续策略 +5. **日报新闻源单一问题** → 前次分析已定位根因(域名映射、RSS 摘要质量),未修复 + +--- + +## CLI 命令速查 + +``` +uv run en-news crawl M1 全量(默认 profile) +uv run en-news crawl --profile 8g_headful M1 headful + proxy +uv run en-news pipeline M2→M6 全链路 +uv run en-news report M7 日报 +uv run en-news search "query" M6 搜索 +uv run en-news mcp-server M8 MCP +bash scripts/domestic_full.sh 全流程(M1→M6→日报) +bash scripts/domestic_crawl_8g.sh 仅 M1 headful +``` diff --git a/crawler/__init__.py b/crawler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/crawler/config.py b/crawler/config.py new file mode 100644 index 0000000..cf64074 --- /dev/null +++ b/crawler/config.py @@ -0,0 +1,59 @@ +"""系统配置加载 + Profile 覆盖。 + +读取 configs/system.yaml,如果设置了 EN_NEWS_PROFILE 环境变量, +则加载 configs/profiles/{name}.yaml 并深度合并覆盖。 +""" + +import logging +import os +from pathlib import Path +from typing import Any + +import yaml + +logger = logging.getLogger(__name__) + +SYSTEM_CONFIG_PATH = Path("configs/system.yaml") +PROFILES_DIR = Path("configs/profiles") + + +def _deep_merge(base: dict, override: dict) -> dict: + """深度合并两个字典,override 的值覆盖 base。""" + result = base.copy() + for key, value in override.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + result[key] = _deep_merge(result[key], value) + else: + result[key] = value + return result + + +def load_system_config() -> dict[str, Any]: + """加载系统配置,自动应用 EN_NEWS_PROFILE 覆盖。 + + Returns: + 合并后的完整配置字典(已应用 profile 覆盖) + """ + if not SYSTEM_CONFIG_PATH.exists(): + raise FileNotFoundError(f"系统配置文件不存在: {SYSTEM_CONFIG_PATH}") + + with open(SYSTEM_CONFIG_PATH, encoding="utf-8") as f: + config: dict[str, Any] = yaml.safe_load(f) or {} + + # ── Profile 覆盖 ────────────────────────────────── + profile_name = os.environ.get("EN_NEWS_PROFILE", "") + if profile_name: + profile_path = PROFILES_DIR / f"{profile_name}.yaml" + if profile_path.exists(): + logger.info("📋 加载 Profile: %s (%s)", profile_name, profile_path) + with open(profile_path, encoding="utf-8") as f: + profile_cfg = yaml.safe_load(f) or {} + config = _deep_merge(config, profile_cfg) + logger.info("📋 Profile 覆盖已应用: proxy=%s, headful=%s, max_memory=%s", + config.get("proxy", {}).get("enabled"), + config.get("crawler", {}).get("headful"), + config.get("crawler", {}).get("max_memory_mb")) + else: + logger.warning("⚠️ Profile 文件不存在: %s", profile_path) + + return config diff --git a/crawler/crawler.py b/crawler/crawler.py new file mode 100644 index 0000000..68f40a3 --- /dev/null +++ b/crawler/crawler.py @@ -0,0 +1,434 @@ +"""Crawl4AI 异步新闻抓取引擎 + +海外服务器运行,约束:内存 ≤ 2GB,串行抓取,单源 ≤ 2 小时。 +所有业务参数从 configs/system.yaml 的 crawler 节读取。 +""" + +import asyncio +import gc +import hashlib +import json +import logging +import os +import re +from datetime import datetime +from pathlib import Path +from urllib.parse import urljoin + +import psutil +import yaml +from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig + +from crawler.models import ArticleItem, CrawlResult, SourceConfig +from crawler.utils import get_news_day + +logger = logging.getLogger(__name__) + +# ── 从配置文件加载常量 ────────────────────────────────── + + +def _load_crawler_config() -> dict: + """从 system.yaml 读取 crawler 配置节(含 Profile 覆盖)""" + try: + from crawler.config import load_system_config + cfg = load_system_config() + return cfg.get("crawler", {}) + except Exception: + logger.warning("读取 system.yaml 失败,使用默认值") + return {} + +_cfg = _load_crawler_config() + +MAX_MEMORY_MB = int(_cfg.get("max_memory_mb", 1800)) +ARTICLE_DELAY_SEC = float(_cfg.get("article_delay_sec", 3.0)) +PAGE_TIMEOUT_MS = int(_cfg.get("page_timeout_sec", 45)) * 1000 +SOURCE_TIMEOUT_SEC = int(_cfg.get("source_timeout_sec", 7200)) +_VIEWPORT_WIDTH = int(_cfg.get("viewport_width", 1024)) +_VIEWPORT_HEIGHT = int(_cfg.get("viewport_height", 768)) +_USER_AGENT = _cfg.get( + "user_agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/131.0.0.0 Safari/537.36", +) +_HEADFUL = bool(_cfg.get("headful", False)) +_XVFB_DISPLAY = _cfg.get("xvfb_display", ":99") + +# ── 代理配置 ────────────────────────────────────────── + + +def _load_proxy_config() -> dict: + """从 system.yaml 读取 proxy 配置节(含 Profile 覆盖)""" + try: + from crawler.config import load_system_config + cfg = load_system_config() + return cfg.get("proxy", {}) + except Exception: + pass + return {} + +_proxy_cfg = _load_proxy_config() +PROXY_ENABLED = bool(_proxy_cfg.get("enabled", False)) +PROXY_URL = _proxy_cfg.get("url", "socks5://127.0.0.1:1080") + +# ── 工具函数 ────────────────────────────────────────── + + +def compute_url_hash(url: str) -> str: + """计算 URL 的 SHA256 前 16 个字符作为短文件名""" + return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16] + + +def _get_memory_mb() -> float: + """获取当前进程内存使用量(MB)""" + proc = psutil.Process(os.getpid()) + return proc.memory_info().rss / (1024 * 1024) + + +def _check_memory(source_id: str) -> None: + """检查内存,超限时警告""" + mem = _get_memory_mb() + if mem > MAX_MEMORY_MB: + logger.warning("[%s] ⚠️ 内存使用 %.0f MB 超过上限 %.0f MB,触发 GC", + source_id, mem, MAX_MEMORY_MB) + gc.collect() + mem_after = _get_memory_mb() + logger.info("[%s] GC 后内存: %.0f MB (回收 %.0f MB)", + source_id, mem_after, mem - mem_after) + +# ── 浏览器配置 ────────────────────────────────────────── + + +def _make_browser_config(source: SourceConfig) -> BrowserConfig: + """浏览器配置:根据 anti_bot_mode 和系统配置选择策略 + + - None: 标准轻量 headless(默认) + - "stealth": 反检测 headless(隐藏自动化特征) + - "headful": 非 headless 模式(有头浏览器,最像真人) + + 代理:如果 system.yaml proxy.enabled=true,浏览器流量走 SOCKS5 代理。 + headful 模式自动设置 DISPLAY 环境变量(支持 xvfb)。 + """ + mode = source.anti_bot_mode + # headless 判定: + # - 系统级 _HEADFUL=true → 强制有头(覆盖所有 source 模式) + # - source mode="headful" → 有头 + # - 其他 → headless + if _HEADFUL: + headless = False # 系统级强制 headful + elif mode == "headful": + headless = False + else: + headless = True + + # 基础反检测参数 + extra_args = [ + "--disable-dev-shm-usage", + "--disable-gpu", + "--no-sandbox", + ] + + # 反检测参数:stealth 模式或系统级 headful 都启用 + if mode == "stealth" or _HEADFUL: + extra_args += [ + "--disable-blink-features=AutomationControlled", + "--disable-features=IsolateOrigins,site-per-process", + ] + + # ── SOCKS5 代理 ────────────────────────────────── + if PROXY_ENABLED and PROXY_URL: + extra_args.append(f"--proxy-server={PROXY_URL}") + logger.info("🌐 浏览器代理已启用: %s", PROXY_URL) + + # ── headful 模式:设置虚拟显示器 ────────────────── + if not headless: + display = os.environ.get("DISPLAY", "") + if not display: + os.environ["DISPLAY"] = _XVFB_DISPLAY + logger.info("🖥️ headful 模式: DISPLAY=%s", _XVFB_DISPLAY) + + return BrowserConfig( + browser_type="chromium", + headless=headless, + viewport_width=_VIEWPORT_WIDTH, + viewport_height=_VIEWPORT_HEIGHT, + verbose=False, + text_mode=headless, # headful 模式下不禁用图片(更像真人) + light_mode=headless, + user_agent=_USER_AGENT, + extra_args=extra_args, + ) + + +def _make_run_config(source: SourceConfig) -> CrawlerRunConfig: + """抓取运行时配置""" + return CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + page_timeout=PAGE_TIMEOUT_MS, + wait_until="domcontentloaded", + scan_full_page=False, + simulate_user=True, + override_navigator=True, + ) + +def _extract_domain(url: str) -> str: + """从 URL 提取域名(去掉 www 前缀)。""" + from urllib.parse import urlparse + host = urlparse(url).hostname or "" + return host.removeprefix("www.").lower() + + +# ── 链接提取 ────────────────────────────────────────── + + +async def _extract_article_urls( + crawler: AsyncWebCrawler, + source: SourceConfig, +) -> list[str]: + """从首页抓取符合 article_url_pattern 的文章链接(增量:跳过已抓取过的 URL)""" + config = _make_run_config(source) + + try: + result = await crawler.arun(url=source.homepage, config=config) + except Exception as e: + logger.error("抓取首页失败 [%s] %s: %s", source.id, source.homepage, e) + return [] + + if not result.success: + logger.error("首页返回失败 [%s]: %s", source.id, result.error_message) + return [] + + html = result.html or "" + pattern = re.compile(r'href=["\']([^"\']*?)["\']', re.IGNORECASE) + raw_urls: list[str] = pattern.findall(html) + + article_url_pattern = re.compile(source.article_url_pattern, re.IGNORECASE) + base_without_fragment = source.homepage.split("#")[0] + base_domain = _extract_domain(source.homepage) + + # 非内容类扩展名 + _SKIP_EXT = re.compile( + r"\.(png|ico|gif|jpg|jpeg|svg|webp|css|js|xml|json|rss|pdf|zip|woff2?|ttf|eot)" + r"([?#]|$)", + re.IGNORECASE, + ) + # 非文章技术路径 + _SKIP_PATH = re.compile( + r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)" + r"[/.]", + re.IGNORECASE, + ) + + article_urls: list[str] = [] + for raw in raw_urls: + raw_stripped = raw.strip() + if raw_stripped.startswith("#") or not raw_stripped: + continue + if raw_stripped.lower().startswith("javascript:"): + continue + + full = urljoin(source.homepage, raw_stripped) + full_clean = full.split("#")[0] + + # 过滤非内容类 URL + if _SKIP_EXT.search(full_clean): + continue + if _SKIP_PATH.search(full_clean): + continue + # 过滤非同域链接(广告、CDN 等) + if _extract_domain(full_clean) != base_domain: + continue + + if full_clean.rstrip("/") == base_without_fragment.rstrip("/"): + continue + if article_url_pattern.search(full_clean): + article_urls.append(full_clean) + + # 本批去重 + seen: set[str] = set() + unique: list[str] = [] + for u in article_urls: + if u not in seen: + seen.add(u) + unique.append(u) + + # 增量过滤:读取已抓取的 url_hash,跳过已存在 URL + today = get_news_day() + index_path = Path(f"data/raw/{source.id}/{today}/index.jsonl") + existing_hashes: set[str] = set() + if index_path.exists(): + with open(index_path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + item = json.loads(line) + existing_hashes.add(item.get("url_hash", "")) + except json.JSONDecodeError: + continue + + filtered: list[str] = [] + skipped = 0 + for u in unique: + uh = compute_url_hash(u) + if uh in existing_hashes: + skipped += 1 + logger.debug("[%s] 跳过已抓取: %s", source.id, u[:80]) + continue + filtered.append(u) + if len(filtered) >= source.max_articles_per_run: + break + + if skipped > 0: + logger.info("[%s] 增量过滤: 跳过 %d 篇已抓取,剩余 %d 篇待抓取", + source.id, skipped, len(filtered)) + + logger.info("[%s] 从首页提取到 %d 个文章链接(去重后 %d,增量后 %d)", + source.id, len(article_urls), len(unique), len(filtered)) + return filtered + +# ── 单篇抓取 ────────────────────────────────────────── + + +async def _crawl_single_article( + crawler: AsyncWebCrawler, + url: str, + source: SourceConfig, + index: int, + total: int, +) -> ArticleItem: + """抓取单篇文章(串行模式)""" + url_hash = compute_url_hash(url) + now = datetime.now() + today = get_news_day() + + article = ArticleItem( + source_id=source.id, + source_name=source.name, + url=url, + url_hash=url_hash, + title="", + crawl_time=now.isoformat(), + html_path="", + status="failed", + ) + + try: + config = _make_run_config(source) + result = await crawler.arun(url=url, config=config) + + if not result.success: + article.error = result.error_message or "Unknown error" + logger.warning("[%s] [%d/%d] 抓取失败: %s", + source.id, index, total, article.error[:100]) + return article + + # 提取标题 + article.title = (result.metadata.get("title") if result.metadata else "") or "" + + # 保存输出 + out_dir = Path(f"data/raw/{source.id}/{today}") + out_dir.mkdir(parents=True, exist_ok=True) + + # HTML + html_path = out_dir / f"{url_hash}.html" + if result.html: + html_path.write_text(result.html, encoding="utf-8") + article.html_path = str(html_path) + + # Markdown + md_path = out_dir / f"{url_hash}.md" + md_content = "" + if hasattr(result, "markdown") and result.markdown: + md_content = str(result.markdown) + md_path.write_text(md_content, encoding="utf-8") + article.md_path = str(md_path) + + article.word_count = len(md_content.split()) if md_content else 0 + article.status = "success" + + # 每篇都打日志方便追踪进度 + mem_mb = _get_memory_mb() + logger.info("[%s] [%d/%d] ✅ %s (%d words, %.0f MB)", + source.id, index, total, article.title[:50], article.word_count, mem_mb) + + except Exception as e: + article.error = str(e) + logger.error("[%s] [%d/%d] 抓取异常: %s", source.id, index, total, e) + + return article + +# ── 单源抓取 ────────────────────────────────────────── + + +async def crawl_source(source: SourceConfig) -> CrawlResult: + """抓取单个新闻源:首页 → 串行抓取每篇文章 + + 约束:内存 ≤ 2GB,超时 ≤ 2 小时 + """ + start_time = datetime.now() + result = CrawlResult( + source_id=source.id, + source_name=source.name, + start_time=start_time.isoformat(), + ) + + mem_start = _get_memory_mb() + logger.info("━━━ 开始 [%s] %s (内存: %.0f MB) ━━━", source.id, source.name, mem_start) + + browser_config = _make_browser_config(source) + + try: + async with asyncio.timeout(SOURCE_TIMEOUT_SEC): + async with AsyncWebCrawler(config=browser_config) as crawler: + # 1. 首页提取链接 + article_urls = await _extract_article_urls(crawler, source) + result.total_found = len(article_urls) + + if not article_urls: + logger.warning("[%s] 未提取到任何文章链接", source.id) + result.end_time = datetime.now().isoformat() + return result + + # 2. 串行抓取每篇文章 + total = len(article_urls) + for i, url in enumerate(article_urls, 1): + _check_memory(source.id) + + article = await _crawl_single_article( + crawler, url, source, index=i, total=total, + ) + + if article.status == "success": + result.total_success += 1 + else: + result.total_failed += 1 + result.articles.append(article) + + # 冷却间隔 + if i < total: + await asyncio.sleep(ARTICLE_DELAY_SEC) + + except TimeoutError: + logger.error("[%s] ⏰ 超时 %.0f 秒(上限 %d 秒),中止", + source.id, (datetime.now() - start_time).total_seconds(), + SOURCE_TIMEOUT_SEC) + result.error = "Source timeout" + except Exception as e: + logger.exception("[%s] 抓取过程异常: %s", source.id, e) + result.error = str(e) + + # 主动回收 + gc.collect() + + result.end_time = datetime.now().isoformat() + elapsed = (datetime.now() - start_time).total_seconds() + mem_end = _get_memory_mb() + logger.info( + "[%s] 完成: 发现 %d / 成功 %d / 失败 %d,耗时 %.1f 秒,内存 %.0f→%.0f MB", + source.id, result.total_found, result.total_success, result.total_failed, + elapsed, mem_start, mem_end, + ) + + return result diff --git a/crawler/loader.py b/crawler/loader.py new file mode 100644 index 0000000..cc1e0d9 --- /dev/null +++ b/crawler/loader.py @@ -0,0 +1,65 @@ +"""加载和管理新闻源配置""" + +import logging +from pathlib import Path + +import yaml + +from crawler.models import SourceConfig + +logger = logging.getLogger(__name__) + +# 默认配置文件路径 +DEFAULT_SOURCES_PATH = Path("configs/sources.yaml") + + +def load_sources( + config_path: Path | None = None, +) -> tuple[list[SourceConfig], dict]: + """从 YAML 加载新闻源配置 + + Args: + config_path: 配置文件路径,默认 configs/sources.yaml + + Returns: + (启用的源列表, settings dict) + """ + path = config_path or DEFAULT_SOURCES_PATH + + if not path.exists(): + raise FileNotFoundError(f"新闻源配置文件不存在: {path}") + + with open(path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + + if not raw or "sources" not in raw: + raise ValueError(f"配置文件格式错误,缺少 'sources' 字段: {path}") + + settings = raw.get("settings", {}) + sources: list[SourceConfig] = [] + + for item in raw["sources"]: + if not isinstance(item, dict): + logger.warning("跳过非字典格式的源配置: %s", item) + continue + + if not item.get("enabled", True): + logger.info("跳过已禁用的源: %s (%s)", item.get("id"), item.get("name")) + continue + + try: + source = SourceConfig(**item) + sources.append(source) + except Exception as e: + logger.error("解析源配置失败: %s - %s", item.get("id"), e) + + logger.info("成功加载 %d 个启用的新闻源(共 %d 个)", len(sources), len(raw["sources"])) + return sources, settings + + +def get_source_by_id(source_id: str, sources: list[SourceConfig]) -> SourceConfig | None: + """按 ID 查找源""" + for s in sources: + if s.id == source_id: + return s + return None diff --git a/crawler/models.py b/crawler/models.py new file mode 100644 index 0000000..dd20cf0 --- /dev/null +++ b/crawler/models.py @@ -0,0 +1,68 @@ +"""爬虫数据模型""" + +from datetime import datetime +from pathlib import Path + +from pydantic import BaseModel, Field + + +class SourceConfig(BaseModel): + """单个新闻源配置""" + + id: str + name: str + enabled: bool = True + homepage: str + article_url_pattern: str + js_render: bool = False + max_articles_per_run: int = 30 + rss_url: str | None = None # RSS/Atom feed URL(优先使用,绕过反爬) + anti_bot_mode: str | None = None # "stealth" | "headful" | None(反爬策略) + + @property + def output_dir(self) -> Path: + """按日期组织的输出目录""" + today = datetime.now().strftime("%Y%m%d") + return Path(f"data/raw/{self.id}/{today}") + + +class ArticleItem(BaseModel): + """单篇已抓取的文章元数据""" + + source_id: str + source_name: str + url: str + url_hash: str + title: str + crawl_time: str # ISO 8601 + publish_time: str = "" # ISO 8601,RSS 源可提取,网页源由 extractor 补充 + html_path: str # 相对路径,如 data/raw/reuters/20260621/abc123.html + md_path: str = "" # Crawl4AI 生成的 Markdown 路径 + word_count: int = 0 + status: str = "success" # success | failed + error: str = "" + + +class CrawlResult(BaseModel): + """单次抓取结果统计""" + + source_id: str + source_name: str + total_found: int = 0 + total_success: int = 0 + total_failed: int = 0 + articles: list[ArticleItem] = Field(default_factory=list) + start_time: str = "" + end_time: str = "" + error: str = "" + + +class PipelineStats(BaseModel): + """一次完整抓取管道的统计""" + + start_time: str = "" + end_time: str = "" + sources_crawled: int = 0 + sources_failed: int = 0 + total_articles: int = 0 + results: list[CrawlResult] = Field(default_factory=list) diff --git a/crawler/orchestrator.py b/crawler/orchestrator.py new file mode 100644 index 0000000..3be568d --- /dev/null +++ b/crawler/orchestrator.py @@ -0,0 +1,126 @@ +"""抓取编排器:串行调度多个新闻源的抓取。支持 RSS 优先、stealth/headful 回退。""" + +import asyncio +import logging +from datetime import datetime + +from crawler.crawler import crawl_source +from crawler.loader import load_sources +from crawler.models import CrawlResult, PipelineStats, SourceConfig +from crawler.rss_crawler import crawl_rss_source +from crawler.storage import write_index_jsonl + +logger = logging.getLogger(__name__) + + +async def _crawl_single_source_with_storage(source: SourceConfig) -> CrawlResult: + """抓取单个源并写入存储。 + + 优先级: + 1. 有 rss_url → RSS 抓取(绕过反爬) + 2. RSS 失败或未配置 → Web 抓取(含 stealth/headful) + """ + result = await crawl_source_smart(source) + if result.total_success > 0: + write_index_jsonl(result) + return result + + +async def crawl_source_smart(source: SourceConfig) -> CrawlResult: + """智能抓取:RSS 优先,Web 回退。 + + RSS 结果判定: + - total_success > 0 → 有新文章,直接返回 + - total_found > 0, success=0 → 增量全部跳过(正常,不浪费 Web 回退) + - total_found == 0 → RSS 真·空结果 + - error 非空 → RSS 失败(网络/解析错),回退 Web + + Web 回退仅在 RSS 失败或未配置时执行。 + """ + rss_url = getattr(source, "rss_url", None) + + # ── 策略 1: RSS(同步,不消耗浏览器资源)── + if rss_url: + logger.info("[%s] 尝试 RSS 抓取: %s", source.id, rss_url) + rss_result = crawl_rss_source(source) + + if rss_result.total_success > 0: + logger.info("[%s] ✅ RSS 成功: %d 篇", source.id, rss_result.total_success) + return rss_result + + if rss_result.error: + # RSS 抓取本身失败(网络错/解析错)→ 回退 Web + logger.warning("[%s] RSS 失败 (%s),回退 Web 抓取 (mode=%s)", + source.id, rss_result.error, source.anti_bot_mode) + elif rss_result.total_found > 0: + # RSS 成功但全部增量跳过 → 正常,不浪费 Web 资源 + logger.info("[%s] RSS 无新文章(%d 条全部已抓取),跳过 Web 回退", + source.id, rss_result.total_found) + return rss_result + else: + # total_found == 0,RSS 返回空 + logger.warning("[%s] RSS 返回空,回退 Web 抓取 (mode=%s)", + source.id, source.anti_bot_mode) + + # ── 策略 2/3: Web 抓取(stealth / headful)── + mode = source.anti_bot_mode or "standard" + logger.info("[%s] 开始 Web 抓取 (mode=%s)", source.id, mode) + web_result = await crawl_source(source) + + return web_result + + +async def crawl_all_sources( + source_filter: str | None = None, +) -> PipelineStats: + """串行抓取所有启用的新闻源(海外服务器内存约束,逐个执行) + + Args: + source_filter: 可选,只抓取指定 source_id + + Returns: + PipelineStats 总体统计 + """ + sources, _settings = load_sources() + + if source_filter: + sources = [s for s in sources if s.id == source_filter] + if not sources: + raise ValueError(f"未找到启用的源: {source_filter}") + + logger.info("══════ 开始串行抓取 %d 个新闻源 ══════", len(sources)) + + start_time = datetime.now() + stats = PipelineStats(start_time=start_time.isoformat(), sources_crawled=0) + + # 串行执行每个源 + for source in sources: + result = await _crawl_single_source_with_storage(source) + + if isinstance(result, Exception): + logger.error("源抓取异常: %s", result) + stats.sources_failed += 1 + else: + stats.sources_crawled += 1 + stats.total_articles += result.total_success + stats.results.append(result) + if result.error: + stats.sources_failed += 1 + + stats.end_time = datetime.now().isoformat() + elapsed = (datetime.now() - start_time).total_seconds() + + logger.info( + "══════ 抓取完成: %d/%d 源成功,共 %d 篇文章,耗时 %.1f 秒 ══════", + stats.sources_crawled - stats.sources_failed, + stats.sources_crawled, + stats.total_articles, + elapsed, + ) + + return stats + + +def run_crawl_sync(source_filter: str | None = None) -> PipelineStats: + """同步包装器,供 CLI 调用""" + return asyncio.run(crawl_all_sources(source_filter=source_filter)) diff --git a/crawler/rss_crawler.py b/crawler/rss_crawler.py new file mode 100644 index 0000000..5c19ffb --- /dev/null +++ b/crawler/rss_crawler.py @@ -0,0 +1,393 @@ +"""RSS/Atom Feed 抓取模块。 + +用于有 RSS 服务的新闻源(如 MarketWatch),绕过网页反爬。 +支持 RSS 2.0、Atom、Google News RSS 三种格式。 + +Google News RSS: 处理标题去来源后缀、链接为 Google 跳转 URL。 +""" + +import hashlib +import logging +import re +from datetime import datetime, timezone +from html import unescape +from pathlib import Path +from typing import Any +from urllib.parse import urljoin +from xml.etree import ElementTree + +import httpx + +from crawler.models import ArticleItem, CrawlResult, SourceConfig +from crawler.utils import get_news_day + +logger = logging.getLogger(__name__) + +# ── HTML 标签清洗 ────────────────────────────────────── + + +def _strip_html(text: str) -> str: + """去除 HTML 标签,保留纯文本。""" + if not text: + return "" + text = unescape(text) + return re.sub(r"<[^>]+>", "", text).strip() + + +# ── Namespace ────────────────────────────────────────── + + +def _ns(tag: str) -> str: + """Atom namespace helper。""" + return f"{{http://www.w3.org/2005/Atom}}{tag}" + + +def _compute_url_hash(url: str) -> str: + return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16] + + +# 非内容类扩展名 — RSS 项也可能包含静态资源 +_SKIP_EXT = re.compile( + r"\.(png|ico|gif|jpg|jpeg|svg|webp|css|js|xml|json|rss|pdf|zip|woff2?|ttf|eot)" + r"([?#]|$)", + re.IGNORECASE, +) + +# 明显非文章路径 +_SKIP_PATH = re.compile( + r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)" + r"[/.]", + re.IGNORECASE, +) + + +def _is_valid_article_url(url: str, source: SourceConfig) -> bool: + """校验 RSS 条目 URL 是否为有效文章链接。 + + 过滤规则: + 1. 静态资源(JS/CSS/图片/字体/数据文件) + 2. 技术路径(manifest/_next/static/robots) + 3. 源 article_url_pattern 不匹配 + 4. 域名不一致(RSS 跨站污染,如 Yahoo Finance RSS 混入 sports/tech) + """ + if _SKIP_EXT.search(url): + logger.debug("RSS 跳过静态资源: %s", url[:80]) + return False + if _SKIP_PATH.search(url): + logger.debug("RSS 跳过技术路径: %s", url[:80]) + return False + + # 源级 article_url_pattern 校验 + pattern = getattr(source, "article_url_pattern", "") + if pattern: + try: + if not re.search(pattern, url, re.IGNORECASE): + logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80]) + return False + except re.error: + pass # 正则异常不阻塞 + + # 域名一致性校验(RSS 跨站污染防护) + # 例外: Google News RSS 的链接是 news.google.com 跳转 URL,跳过域名校验 + rss_url = getattr(source, "rss_url", "") + if rss_url and "news.google.com" not in rss_url: + from urllib.parse import urlparse + item_domain = (urlparse(url).hostname or "").removeprefix("www.") + hp_domain = (urlparse(source.homepage).hostname or "").removeprefix("www.") + + if hp_domain and item_domain: + if item_domain != hp_domain and not item_domain.endswith("." + hp_domain): + logger.debug( + "RSS 跳过跨站 URL: %s (domain=%s, expected=%s)", + url[:80], item_domain, hp_domain, + ) + return False + + return True + + +def _parse_date(date_str: str | None) -> str: + """尝试解析常见日期格式,返回 ISO 8601 字符串。""" + if not date_str: + return datetime.now().isoformat() + + formats = [ + "%a, %d %b %Y %H:%M:%S %z", # RFC 2822 + "%a, %d %b %Y %H:%M:%S %Z", + "%Y-%m-%dT%H:%M:%S%z", # ISO 8601 + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", + ] + for fmt in formats: + try: + return datetime.strptime(date_str, fmt).isoformat() + except ValueError: + continue + return date_str + + +# ── Google News RSS 标题清洗 ──────────────────────────── + +_KNOWN_SOURCES = [ + "Reuters", "WSJ", "Bloomberg", "CNBC", "Financial Times", + "MarketWatch", "Barron's", "Yahoo Finance", "Investing.com", + "Seeking Alpha", "The Economist", "ForexLive", "ZeroHedge", +] + + +def _clean_google_news_title(title: str) -> str: + """Google News RSS 标题格式: 'Article Title - SourceName' → 去掉来源后缀。""" + for src in _KNOWN_SOURCES: + suffix = f" - {src}" + if title.endswith(suffix): + return title[: -len(suffix)].strip() + # 通用回退:最后一个 " - " 之后可能是来源 + last_dash = title.rfind(" - ") + if last_dash > 0: + suffix = title[last_dash + 3:] + if len(suffix) < 30 and not suffix.startswith("http"): + return title[:last_dash].strip() + return title + + +# ── RSS/Atom 解析 ───────────────────────────────────── + + +def _extract_rss_items(xml_text: str) -> list[dict[str, Any]]: + """从 RSS/Atom XML 中提取文章条目。 + + 自动识别:RSS 2.0、Google News RSS、Atom。 + """ + # 清理可能的 BOM + if xml_text.startswith(""): + xml_text = xml_text[1:] + + root = ElementTree.fromstring(xml_text) + items: list[dict[str, Any]] = [] + + # 检测是否为 Google News RSS + first_item = root.find(".//item") + is_google_news = False + if first_item is not None: + source_el = first_item.find("source") + if source_el is not None and source_el.text: + is_google_news = True + + # RSS 2.0 / Google News RSS: channel/item + for item in root.iter("item"): + title = _strip_html(_text(item, "title")) + link = _text(item, "link") + description = _strip_html(_text(item, "description")) + pub_date = _text(item, "pubDate") + + if is_google_news and title: + title = _clean_google_news_title(title) + # 描述通常比标题更长,含摘要 + if description and len(description) > len(title): + summary = description + else: + summary = title + else: + summary = description or title + + if link and title: + items.append({ + "title": title, + "url": link.strip(), + "summary": summary, + "publish_time": _parse_date(pub_date), + }) + + if items: + tag = "Google News RSS" if is_google_news else "RSS 2.0" + logger.debug("%s: 提取 %d 条", tag, len(items)) + return items + + # Atom: feed/entry + for entry in root.iter(_ns("entry")): + title = _strip_html(_text(entry, _ns("title"))) + link = _attr(entry, _ns("link"), "href") + summary = _strip_html(_text(entry, _ns("summary"))) + updated = _text(entry, _ns("updated")) + if link and title: + items.append({ + "title": title, + "url": link.strip(), + "summary": summary or title, + "publish_time": _parse_date(updated), + }) + + logger.debug("Atom: 提取 %d 条", len(items)) + return items + + +def _text(element: ElementTree.Element, tag: str) -> str: + """安全获取子元素文本。""" + child = element.find(tag) + return (child.text or "").strip() if child is not None and child.text else "" + + +def _attr(element: ElementTree.Element, tag: str, attr: str) -> str: + """安全获取子元素属性。""" + child = element.find(tag) + return (child.get(attr) or "").strip() if child is not None else "" + + +# ── 主抓取接口 ─────────────────────────────────────── + + +def crawl_rss_source(source: SourceConfig) -> CrawlResult: + """通过 RSS/Atom Feed 抓取单个新闻源。 + + 支持 Google News RSS 代理模式: + - rss_url 为 news.google.com 时自动识别 + - 标题自动去来源后缀 + - 不抓取原文(跳过 Crawl4AI),直接用 RSS 摘要入库 + + Args: + source: 源配置(需含 rss_url 字段) + + Returns: + CrawlResult + """ + rss_url = getattr(source, "rss_url", None) + if not rss_url: + return CrawlResult( + source_id=source.id, + source_name=source.name, + error="源未配置 rss_url", + ) + + today = get_news_day() + out_dir = Path(f"data/raw/{source.id}/{today}") + out_dir.mkdir(parents=True, exist_ok=True) + + start_time = datetime.now() + articles: list[ArticleItem] = [] + crawl_time = start_time.isoformat() + + try: + resp = httpx.get( + rss_url, + follow_redirects=True, + timeout=30.0, + headers={ + "User-Agent": ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36" + ), + "Accept": "application/rss+xml, application/xml, text/xml, */*", + }, + ) + resp.raise_for_status() + xml_text = resp.text + except Exception as e: + logger.error("[%s] RSS 抓取失败: %s", source.id, e) + return CrawlResult( + source_id=source.id, + source_name=source.name, + start_time=crawl_time, + end_time=datetime.now().isoformat(), + error=str(e), + ) + + # 解析 RSS + try: + items = _extract_rss_items(xml_text) + except Exception as e: + logger.error("[%s] RSS 解析失败: %s", source.id, e) + return CrawlResult( + source_id=source.id, + source_name=source.name, + start_time=crawl_time, + end_time=datetime.now().isoformat(), + error=f"RSS 解析失败: {e}", + ) + + if not items: + logger.warning("[%s] RSS 返回 0 条", source.id) + return CrawlResult( + source_id=source.id, + source_name=source.name, + total_found=0, + start_time=crawl_time, + end_time=datetime.now().isoformat(), + ) + + # 过滤已存在的 URL(增量) + index_path = out_dir / "index.jsonl" + existing_hashes: set[str] = _load_existing_hashes(index_path) + + # 构造 ArticleItem + success = 0 + for item in items[:source.max_articles_per_run]: + url_hash = _compute_url_hash(item["url"]) + if url_hash in existing_hashes: + logger.debug("[%s] 跳过 RSS 已抓取: %s", source.id, item["url"][:80]) + continue + + # URL 有效性校验(静态资源 / 跨站污染 / 不匹配 article_url_pattern) + if not _is_valid_article_url(item["url"], source): + continue + + # 保存文章摘要为 Markdown + md_content = f"# {item['title']}\n\n" + md_content += f"**来源**: {source.name}\n\n" + md_content += f"**发布时间**: {item['publish_time']}\n\n" + md_content += f"**原文链接**: {item['url']}\n\n" + md_content += f"{item['summary']}\n" + + md_path = out_dir / f"{url_hash}.md" + md_path.write_text(md_content, encoding="utf-8") + html_path = out_dir / f"{url_hash}.html" + html_content = f"{item['title']}{item['summary']}" + html_path.write_text(html_content, encoding="utf-8") + + article = ArticleItem( + source_id=source.id, + source_name=source.name, + url=item["url"], + url_hash=url_hash, + title=item["title"], + crawl_time=crawl_time, + publish_time=item["publish_time"], + html_path=str(html_path), + md_path=str(md_path), + word_count=len(item["summary"].split()), + status="success", + ) + articles.append(article) + existing_hashes.add(url_hash) + success += 1 + + logger.info("[%s] RSS 抓取: %d 条新文章(总共 %d 条)", + source.id, success, len(items)) + + return CrawlResult( + source_id=source.id, + source_name=source.name, + total_found=len(items), + total_success=success, + articles=articles, + start_time=crawl_time, + end_time=datetime.now().isoformat(), + ) + + +def _load_existing_hashes(index_path: Path) -> set[str]: + """读取 index.jsonl 中已有 url_hash。""" + hashes: set[str] = set() + if not index_path.exists(): + return hashes + import json + with open(index_path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + hashes.add(json.loads(line).get("url_hash", "")) + except json.JSONDecodeError: + continue + return hashes diff --git a/crawler/storage.py b/crawler/storage.py new file mode 100644 index 0000000..9304dc9 --- /dev/null +++ b/crawler/storage.py @@ -0,0 +1,95 @@ +"""存储管理:index.jsonl 读写、输出目录管理""" + +import json +import logging +from pathlib import Path + +from crawler.models import ArticleItem, CrawlResult +from crawler.utils import get_news_day + +logger = logging.getLogger(__name__) + + +def write_index_jsonl(result: CrawlResult) -> Path: + """将单源抓取结果写入 index.jsonl + + Args: + result: 单源抓取结果 + + Returns: + index 文件路径 + """ + today = get_news_day() + out_dir = Path(f"data/raw/{result.source_id}/{today}") + out_dir.mkdir(parents=True, exist_ok=True) + + index_path = out_dir / "index.jsonl" + + # 追加写入(同一天多次抓取合并) + existing: set[str] = set() + if index_path.exists(): + with open(index_path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + item = json.loads(line) + existing.add(item.get("url_hash", "")) + except json.JSONDecodeError: + continue + + written = 0 + with open(index_path, "a", encoding="utf-8") as f: + for article in result.articles: + if article.status != "success": + continue + if article.url_hash in existing: + continue # 跳过已存在的 + + record = article.model_dump() + f.write(json.dumps(record, ensure_ascii=False) + "\n") + existing.add(article.url_hash) + written += 1 + + logger.info("[%s] index.jsonl 写入 %d 条(跳过重复 %d 条)", + result.source_id, written, + len(result.articles) - written) + return index_path + + +def load_index( + source_id: str, + date_str: str | None = None, +) -> list[ArticleItem]: + """读取指定源/日期的 index.jsonl + + Args: + source_id: 新闻源 ID + date_str: 日期字符串 YYYYMMDD,默认当前新闻日 + + Returns: + ArticleItem 列表 + """ + if date_str is None: + date_str = get_news_day() + + index_path = Path(f"data/raw/{source_id}/{date_str}/index.jsonl") + + if not index_path.exists(): + logger.warning("index 文件不存在: %s", index_path) + return [] + + articles: list[ArticleItem] = [] + with open(index_path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + data = json.loads(line) + articles.append(ArticleItem(**data)) + except (json.JSONDecodeError, Exception) as e: + logger.warning("解析 index 行失败: %s", e) + + return articles diff --git a/crawler/utils.py b/crawler/utils.py new file mode 100644 index 0000000..3a9d172 --- /dev/null +++ b/crawler/utils.py @@ -0,0 +1,51 @@ +"""爬虫工具函数""" + +import logging +from datetime import datetime, timedelta +from pathlib import Path + +import yaml + +logger = logging.getLogger(__name__) + +# 默认日切分小时(凌晨) +DEFAULT_CUTOFF_HOUR = 6 + + +def _load_cutoff_hour() -> int: + """从 system.yaml 读取日切分小时""" + config_path = Path("configs/system.yaml") + if config_path.exists(): + try: + with open(config_path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + return int(raw.get("schedule", {}).get("day_cutoff_hour", DEFAULT_CUTOFF_HOUR)) + except Exception: + pass + return DEFAULT_CUTOFF_HOUR + + +def get_news_day(now: datetime | None = None) -> str: + """获取当前新闻日(YYYYMMDD) + + 规则:当天 06:00 到次日 05:59 属于同一个新闻日。 + 例如 2026-06-19 04:00 → "20260618",07:00 → "20260619" + + Args: + now: 参考时间,默认当前时间 + + Returns: + 新闻日字符串 YYYYMMDD + """ + if now is None: + now = datetime.now() + + cutoff = _load_cutoff_hour() + + if now.hour < cutoff: + # 凌晨 0:00 - 5:59,属于前一天 + news_date = now - timedelta(days=1) + else: + news_date = now + + return news_date.strftime("%Y%m%d") diff --git a/dedup/__init__.py b/dedup/__init__.py new file mode 100644 index 0000000..e175536 --- /dev/null +++ b/dedup/__init__.py @@ -0,0 +1,53 @@ +"""三层新闻去重模块 (M3)。 + +公共 API: + - Deduper: 主类(check / ingest / stats) + - FingerprintStore: SQLite 指纹库(底层,通常无需直接用) + - DedupResult / DedupLayer / DedupStats / Fingerprint: 数据模型 + - simhash64 / hamming / content_hash / normalize_content: 指纹算法 + - dedup_all_sources / dedup_source: 批量去重管道 +""" + +from dedup.deduper import ( + DEFAULT_TIME_WINDOW_DAYS, + Deduper, + article_to_fingerprint, +) +from dedup.hasher import ( + DEFAULT_HAMMING_THRESHOLD, + NGRAM_SIZE, + SIMHASH_BITS, + content_hash, + hamming, + normalize_content, + simhash64, +) +from dedup.models import DedupLayer, DedupResult, DedupStats, Fingerprint +from dedup.pipeline import dedup_all_sources, dedup_source +from dedup.store import DEFAULT_DB_PATH, FingerprintStore + +__all__ = [ + # 主类 + "Deduper", + "FingerprintStore", + # 管道 + "dedup_all_sources", + "dedup_source", + # 模型 + "DedupLayer", + "DedupResult", + "DedupStats", + "Fingerprint", + # 指纹算法 + "article_to_fingerprint", + "content_hash", + "hamming", + "normalize_content", + "simhash64", + # 常量 + "DEFAULT_DB_PATH", + "DEFAULT_HAMMING_THRESHOLD", + "DEFAULT_TIME_WINDOW_DAYS", + "NGRAM_SIZE", + "SIMHASH_BITS", +] diff --git a/dedup/deduper.py b/dedup/deduper.py new file mode 100644 index 0000000..82d7a02 --- /dev/null +++ b/dedup/deduper.py @@ -0,0 +1,178 @@ +"""三层去重主流程。 + +调用顺序: check / ingest 内部按 L1 → L2 → L3 顺序判定,任意层命中即返回。 + +Deduper 不要求线程安全;批处理串行调用即可。 +""" + +import logging +from datetime import datetime +from pathlib import Path + +import yaml + +from dedup.hasher import DEFAULT_HAMMING_THRESHOLD, content_hash, hamming, simhash64 +from dedup.models import DedupLayer, DedupResult, DedupStats, Fingerprint +from dedup.store import DEFAULT_DB_PATH, FingerprintStore +from extractor.models import ProcessedArticle + +logger = logging.getLogger(__name__) + +# 默认时间窗口(±N 天) +DEFAULT_TIME_WINDOW_DAYS = 30 + + +def _load_dedup_config() -> dict: + """从 system.yaml 加载去重配置。""" + config_path = Path("configs/system.yaml") + if config_path.exists(): + try: + with open(config_path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + return raw.get("dedup", {}) + except Exception: + logger.warning("加载 dedup 配置失败,使用默认值") + return {} + + +def _publish_date(article: ProcessedArticle) -> str | None: + """从 ProcessedArticle.publish_time 取 YYYY-MM-DD 字符串。""" + if not article.publish_time: + return None + # publish_time 格式为 ISO 8601,如 "2026-06-21T10:30:00" + try: + return article.publish_time[:10] + except (IndexError, TypeError): + return None + + +def article_to_fingerprint(article: ProcessedArticle) -> Fingerprint: + """构造 Fingerprint(用于 ingest 写入或对外只读)。""" + return Fingerprint( + url_hash=article.url_hash, + content_hash=content_hash(article.content), + simhash=simhash64(article.content), + source_id=article.source_id, + url=article.url, + title=article.title, + publish_date=_publish_date(article), + ingested_at=datetime.now(), + ) + + +class Deduper: + """三层去重器。 + + 构造完毕后: + - check(article) 仅判断,不写入 + - ingest(article) 判断,不重复则写入指纹库,返回结果 + """ + + def __init__( + self, + db_path: str | Path = DEFAULT_DB_PATH, + simhash_threshold: int | None = None, + time_window_days: int | None = None, + ) -> None: + config = _load_dedup_config() + + self.store = FingerprintStore(db_path) + self.simhash_threshold = ( + simhash_threshold + if simhash_threshold is not None + else config.get("hamming_distance_threshold", DEFAULT_HAMMING_THRESHOLD) + ) + self.time_window_days = ( + time_window_days + if time_window_days is not None + else config.get("simhash_window_days", DEFAULT_TIME_WINDOW_DAYS) + ) + + def close(self) -> None: + self.store.close() + + def __enter__(self) -> "Deduper": + return self + + def __exit__(self, *_: object) -> None: + self.close() + + # ------------------------------------------------------------------ # + # 公共 API + # ------------------------------------------------------------------ # + + def check(self, article: ProcessedArticle) -> DedupResult: + """三层判重(只读,不写入指纹库)。""" + fp = article_to_fingerprint(article) + + # L1: URL hash + existing = self.store.get_by_url_hash(fp.url_hash) + if existing is not None: + return DedupResult( + url_hash=fp.url_hash, + is_duplicate=True, + matched_layer=DedupLayer.URL, + matched_url_hash=existing.url_hash, + matched_url=existing.url, + matched_title=existing.title, + ) + + # L2: 内容 hash + existing = self.store.find_by_content_hash(fp.content_hash) + if existing is not None: + return DedupResult( + url_hash=fp.url_hash, + is_duplicate=True, + matched_layer=DedupLayer.CONTENT, + matched_url_hash=existing.url_hash, + matched_url=existing.url, + matched_title=existing.title, + ) + + # L3: SimHash 模糊 + candidates = self.store.candidates_for_simhash( + fp.publish_date, self.time_window_days + ) + best_dist: int | None = None + best_match: Fingerprint | None = None + for c in candidates: + d = hamming(fp.simhash, c.simhash) + if d <= self.simhash_threshold and (best_dist is None or d < best_dist): + best_dist = d + best_match = c + if d == 0: # 不可能更近,提前结束 + break + + if best_match is not None: + return DedupResult( + url_hash=fp.url_hash, + is_duplicate=True, + matched_layer=DedupLayer.SIMHASH, + matched_url_hash=best_match.url_hash, + matched_url=best_match.url, + matched_title=best_match.title, + hamming_distance=best_dist, + ) + + return DedupResult(url_hash=fp.url_hash, is_duplicate=False) + + def ingest(self, article: ProcessedArticle) -> DedupResult: + """判重 + 不重复则入库。""" + result = self.check(article) + if not result.is_duplicate: + fp = article_to_fingerprint(article) + self.store.upsert(fp) + logger.debug("指纹入库: %s %s", fp.url_hash, fp.title[:40]) + else: + logger.debug("命中重复: %s", result.short_summary()) + return result + + def stats(self) -> DedupStats: + """指纹库统计信息。""" + lo, hi = self.store.date_range() + return DedupStats( + total=self.store.count(), + by_source=self.store.count_by_source(), + earliest=lo, + latest=hi, + ) diff --git a/dedup/hasher.py b/dedup/hasher.py new file mode 100644 index 0000000..7f98889 --- /dev/null +++ b/dedup/hasher.py @@ -0,0 +1,89 @@ +"""三层去重的指纹算法。 + +核心: + - normalize_content: 把 content 折叠成纯净文本,用于跨源比对 + - content_hash: normalize 后 SHA1[:16] + - simhash64: 字符 3-gram + md5 加权累加,产出 64 位无符号整数 + - hamming: 两个 SimHash 的汉明距离 + +设计取舍: + SimHash 的"分词"用字符 3-gram 而非英文分词库。理由: + 1. 字符 3-gram 对英文和中文同样有效,无需外部 NLP 依赖 + 2. 英文字符级 3-gram 天然捕获词根、前缀、后缀信息 + 3. 跨语言场景(英文源可能引用中文/日文公司名)字符级更鲁棒 +""" + +import hashlib +import unicodedata + +# 64 位 SimHash 位宽 +SIMHASH_BITS = 64 +SIMHASH_MASK = (1 << SIMHASH_BITS) - 1 + +# 默认 SimHash 汉明距离阈值(≤ 此值视为重复) +DEFAULT_HAMMING_THRESHOLD = 3 + +# 字符 n-gram 长度 +NGRAM_SIZE = 3 + + +def normalize_content(text: str) -> str: + """把 content 折叠成"无空白无标点"形式,用于 L2 内容 hash 与 SimHash 输入。 + + 使用 Unicode 类别判断: + - P* Punctuation(所有中英文标点) + - Z* Separator(空格 / 行 / 段分隔符) + - C* Control(NUL / 换行控制等) + 保留 L*(字母)、N*(数字)、S*(符号,如 +/-、% 等),以及 CJK 字符。 + """ + if not text: + return "" + return "".join( + ch for ch in text if unicodedata.category(ch)[0] not in ("P", "Z", "C") + ) + + +def content_hash(text: str) -> str: + """对 normalize_content(text) 做 SHA1,取前 16 hex 字符。""" + norm = normalize_content(text) + return hashlib.sha1(norm.encode("utf-8")).hexdigest()[:16] + + +def _ngrams(text: str, n: int = NGRAM_SIZE) -> list[str]: + """字符级 n-gram。文本短于 n 时,直接整体作为单个 token。""" + if len(text) < n: + return [text] if text else [] + return [text[i : i + n] for i in range(len(text) - n + 1)] + + +def simhash64(text: str) -> int: + """64 位 SimHash。返回无符号整数,空文本返回 0。""" + norm = normalize_content(text) + if not norm: + return 0 + + grams = _ngrams(norm) + if not grams: + return 0 + + v = [0] * SIMHASH_BITS + for gram in grams: + h = int(hashlib.md5(gram.encode("utf-8"), usedforsecurity=False).hexdigest(), 16) + # 取低 64 位 + h64 = h & SIMHASH_MASK + for i in range(SIMHASH_BITS): + if (h64 >> i) & 1: + v[i] += 1 + else: + v[i] -= 1 + + fp = 0 + for i in range(SIMHASH_BITS): + if v[i] > 0: + fp |= 1 << i + return fp + + +def hamming(a: int, b: int) -> int: + """两个 SimHash 的汉明距离。""" + return bin((a ^ b) & SIMHASH_MASK).count("1") diff --git a/dedup/models.py b/dedup/models.py new file mode 100644 index 0000000..2447606 --- /dev/null +++ b/dedup/models.py @@ -0,0 +1,57 @@ +"""三层去重模块的数据模型。""" + +from datetime import datetime +from enum import StrEnum + +from pydantic import BaseModel, Field + + +class DedupLayer(StrEnum): + """命中去重的层。""" + + URL = "url" # L1: 完全相同 URL + CONTENT = "content" # L2: 标准化后 content 完全一致 + SIMHASH = "simhash" # L3: SimHash 汉明距离 ≤ 阈值 + + +class Fingerprint(BaseModel): + """单篇文章的指纹记录,持久化到 SQLite。""" + + url_hash: str = Field(..., description="主键,与 ProcessedArticle.url_hash 一致") + content_hash: str = Field(..., description="标准化 content 的 SHA1[:16]") + simhash: int = Field(..., description="64 位 SimHash 整数(无符号)") + source_id: str + url: str + title: str + publish_date: str | None = Field(default=None, description="YYYY-MM-DD,用于时间窗口") + ingested_at: datetime = Field(default_factory=datetime.now) + + +class DedupResult(BaseModel): + """对单篇文章的判重结果。""" + + url_hash: str + is_duplicate: bool + matched_layer: DedupLayer | None = None + matched_url_hash: str | None = None + matched_url: str | None = None + matched_title: str | None = None + hamming_distance: int | None = Field( + default=None, description="仅 SimHash 层有值" + ) + + def short_summary(self) -> str: + if not self.is_duplicate: + return f"[UNIQUE] {self.url_hash}" + layer = self.matched_layer.value if self.matched_layer else "?" + extra = f" hd={self.hamming_distance}" if self.hamming_distance is not None else "" + return f"[DUP/{layer}] {self.url_hash} ~ {self.matched_url_hash}{extra}" + + +class DedupStats(BaseModel): + """指纹库统计。""" + + total: int = 0 + by_source: dict[str, int] = Field(default_factory=dict) + earliest: str | None = None + latest: str | None = None diff --git a/dedup/pipeline.py b/dedup/pipeline.py new file mode 100644 index 0000000..6ecab96 --- /dev/null +++ b/dedup/pipeline.py @@ -0,0 +1,226 @@ +"""批量去重管道:扫描 processed 目录 → 判重 → 唯一条目写入 deduped。 + +输入: data/processed/{source_id}/{YYYYMMDD}/{url_hash}.json +输出: data/deduped/{YYYYMMDD}/uniques/{url_hash}.json +""" + +import json +import logging +from datetime import datetime +from pathlib import Path + +from crawler.utils import get_news_day +from dedup.deduper import Deduper +from dedup.models import DedupResult +from extractor.models import ProcessedArticle + +logger = logging.getLogger(__name__) + + +def _get_processed_sources(base_dir: str = "data/processed") -> list[str]: + """扫描 data/processed/ 下所有源 ID。 + + Args: + base_dir: processed 数据根目录 + + Returns: + 源 ID 列表 + """ + raw_path = Path(base_dir) + if not raw_path.exists(): + return [] + return sorted([ + d.name for d in raw_path.iterdir() + if d.is_dir() and not d.name.startswith(".") + ]) + + +def _load_processed_articles( + source_id: str, + date_str: str, +) -> list[ProcessedArticle]: + """加载指定源/日期的已处理文章。 + + Args: + source_id: 新闻源 ID + date_str: 日期 YYYYMMDD + + Returns: + ProcessedArticle 列表 + """ + base_dir = Path(f"data/processed/{source_id}/{date_str}") + if not base_dir.exists(): + return [] + + articles: list[ProcessedArticle] = [] + for json_file in sorted(base_dir.glob("*.json")): + # 跳过 index.jsonl + if json_file.name == "index.jsonl": + continue + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + articles.append(ProcessedArticle(**data)) + except (json.JSONDecodeError, Exception) as e: + logger.warning("解析 processed JSON 失败 %s: %s", json_file, e) + + return articles + + +def dedup_source( + source_id: str, + deduper: Deduper, + date_str: str | None = None, +) -> dict: + """对单个源的已处理文章执行去重。 + + Args: + source_id: 新闻源 ID + deduper: 去重器实例 + date_str: 日期 YYYYMMDD,默认当前新闻日 + + Returns: + 统计 dict + """ + if date_str is None: + date_str = get_news_day() + + logger.info("━━━ 去重 [%s] %s ━━━", source_id, date_str) + + articles = _load_processed_articles(source_id, date_str) + if not articles: + logger.warning("[%s] %s 无待处理文章", source_id, date_str) + return {"source_id": source_id, "total": 0, "unique": 0, "duplicate": 0} + + # 输出目录 + out_dir = Path(f"data/deduped/{date_str}/uniques") + out_dir.mkdir(parents=True, exist_ok=True) + + unique_count = 0 + dup_count = 0 + + for article in articles: + result = deduper.ingest(article) + + if result.is_duplicate: + dup_count += 1 + logger.debug("[%s] 🔁 %s → L%d: %s", + source_id, + article.title[:40], + _layer_num(result), + result.short_summary()) + else: + unique_count += 1 + # 写入唯一条目 + out_file = out_dir / f"{article.url_hash}.json" + out_file.write_text( + article.model_dump_json(indent=2, ensure_ascii=False), + encoding="utf-8", + ) + logger.debug("[%s] ✅ %s (%d words)", + source_id, article.title[:40], article.word_count) + + logger.info("[%s] 去重完成: 唯一 %d / 重复 %d / 总计 %d", + source_id, unique_count, dup_count, len(articles)) + + return { + "source_id": source_id, + "total": len(articles), + "unique": unique_count, + "duplicate": dup_count, + } + + +def _layer_num(result: DedupResult) -> int: + """DedupResult → 命中层编号。""" + if result.matched_layer is None: + return 0 + mapping = {"url": 1, "content": 2, "simhash": 3} + return mapping.get(result.matched_layer.value, 0) + + +def dedup_all_sources( + source_filter: str | None = None, + date_str: str | None = None, +) -> dict: + """对所有源的已处理文章执行去重。 + + Args: + source_filter: 可选,只处理指定源 + date_str: 日期,默认当前新闻日 + + Returns: + 统计 dict + """ + if date_str is None: + date_str = get_news_day() + + start_time = datetime.now() + + if source_filter: + sources = [source_filter] if source_filter in _get_processed_sources() else [] + else: + sources = _get_processed_sources() + + logger.info("══════ 开始去重 %d 个源,日期: %s ══════", len(sources), date_str) + + total_unique = 0 + total_dup = 0 + total_articles = 0 + + with Deduper() as deduper: + for src in sources: + result = dedup_source(src, deduper, date_str) + total_articles += result["total"] + total_unique += result["unique"] + total_dup += result["duplicate"] + + # 输出 dedup 索引 + _write_dedup_index(deduper, date_str, total_unique) + + elapsed = (datetime.now() - start_time).total_seconds() + logger.info("══════ 去重完成: 唯一 %d / 重复 %d / 总计 %d,耗时 %.1f 秒 ══════", + total_unique, total_dup, total_articles, elapsed) + + return { + "sources_processed": len(sources), + "total_articles": total_articles, + "unique": total_unique, + "duplicate": total_dup, + "elapsed_sec": elapsed, + "date": date_str, + } + + +def _write_dedup_index( + deduper: Deduper, + date_str: str, + unique_count: int, +) -> None: + """写出去重索引文件。 + + Args: + deduper: 去重器实例 + date_str: 日期 + unique_count: 唯一文章数 + """ + out_dir = Path(f"data/deduped/{date_str}") + out_dir.mkdir(parents=True, exist_ok=True) + + stats = deduper.stats() + + index_data = { + "date": date_str, + "unique_articles": unique_count, + "fingerprint_db_total": stats.total, + "fingerprint_db_by_source": stats.by_source, + "fingerprint_db_earliest": stats.earliest, + "fingerprint_db_latest": stats.latest, + "generated_at": datetime.now().isoformat(), + } + + index_path = out_dir / "index.json" + index_path.write_text( + json.dumps(index_data, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + logger.info("去重索引已写入: %s", index_path) diff --git a/dedup/store.py b/dedup/store.py new file mode 100644 index 0000000..55d791d --- /dev/null +++ b/dedup/store.py @@ -0,0 +1,177 @@ +"""SQLite 指纹存储。 + +注意: SimHash 是 64 位无符号整数,SQLite INTEGER 是 64 位有符号 +(范围 [-2^63, 2^63-1])。直接存可能溢出/转负数,虽然 XOR 仍然 +正确但语义混乱。这里统一存为 16 位 hex TEXT,避免符号问题。 +""" + +import logging +import sqlite3 +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +from dedup.models import Fingerprint + +logger = logging.getLogger(__name__) + +DEFAULT_DB_PATH = Path("data/dedup/fingerprints.sqlite3") + +_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS fingerprints ( + url_hash TEXT PRIMARY KEY, + content_hash TEXT NOT NULL, + simhash_hex TEXT NOT NULL, + source_id TEXT NOT NULL, + url TEXT NOT NULL, + title TEXT NOT NULL, + publish_date TEXT, + ingested_at TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_content_hash ON fingerprints(content_hash); +CREATE INDEX IF NOT EXISTS idx_publish_date ON fingerprints(publish_date); +CREATE INDEX IF NOT EXISTS idx_source_id ON fingerprints(source_id); +""" + + +def _to_hex(simhash: int) -> str: + """64 位无符号整数 → 16 位 hex 字符串。""" + return f"{simhash:016x}" + + +def _from_hex(hex_str: str) -> int: + """16 位 hex 字符串 → 64 位无符号整数。""" + return int(hex_str, 16) + + +def _row_to_fp(row: sqlite3.Row) -> Fingerprint: + """sqlite3.Row → Fingerprint 模型。""" + return Fingerprint( + url_hash=row["url_hash"], + content_hash=row["content_hash"], + simhash=_from_hex(row["simhash_hex"]), + source_id=row["source_id"], + url=row["url"], + title=row["title"], + publish_date=row["publish_date"], + ingested_at=datetime.fromisoformat(row["ingested_at"]), + ) + + +class FingerprintStore: + """SQLite 指纹库。线程不安全(每个线程请新建实例)。""" + + def __init__(self, db_path: str | Path = DEFAULT_DB_PATH) -> None: + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + self._conn: sqlite3.Connection = sqlite3.connect( + self.db_path, isolation_level=None + ) + self._conn.row_factory = sqlite3.Row + self._conn.executescript(_SCHEMA_SQL) + logger.debug("打开指纹库: %s", self.db_path) + + def close(self) -> None: + self._conn.close() + + def __enter__(self) -> "FingerprintStore": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + # ------------------------------------------------------------------ # + # 查询 + # ------------------------------------------------------------------ # + + def get_by_url_hash(self, url_hash: str) -> Fingerprint | None: + """按 url_hash 精确查询。""" + row = self._conn.execute( + "SELECT * FROM fingerprints WHERE url_hash = ?", (url_hash,) + ).fetchone() + return _row_to_fp(row) if row else None + + def find_by_content_hash(self, content_hash: str) -> Fingerprint | None: + """返回任一 content_hash 匹配项。""" + row = self._conn.execute( + "SELECT * FROM fingerprints WHERE content_hash = ? LIMIT 1", + (content_hash,), + ).fetchone() + return _row_to_fp(row) if row else None + + def candidates_for_simhash( + self, + publish_date: str | None, + window_days: int, + ) -> list[Fingerprint]: + """返回 publish_date ± window_days 内的指纹候选。 + + publish_date 为 None 时,不限定窗口(返回全部,慎用)。 + """ + if publish_date is None or window_days < 0: + rows = self._conn.execute("SELECT * FROM fingerprints").fetchall() + return [_row_to_fp(r) for r in rows] + + try: + center = datetime.strptime(publish_date, "%Y-%m-%d") + except ValueError: + logger.debug("publish_date 不可解析: %r,退化为全表扫描", publish_date) + rows = self._conn.execute("SELECT * FROM fingerprints").fetchall() + return [_row_to_fp(r) for r in rows] + + lo = (center - timedelta(days=window_days)).strftime("%Y-%m-%d") + hi = (center + timedelta(days=window_days)).strftime("%Y-%m-%d") + rows = self._conn.execute( + "SELECT * FROM fingerprints " + "WHERE publish_date IS NULL OR (publish_date >= ? AND publish_date <= ?)", + (lo, hi), + ).fetchall() + return [_row_to_fp(r) for r in rows] + + def count(self) -> int: + """指纹总数。""" + return self._conn.execute("SELECT COUNT(*) FROM fingerprints").fetchone()[0] + + def count_by_source(self) -> dict[str, int]: + """按 source_id 统计。""" + rows = self._conn.execute( + "SELECT source_id, COUNT(*) AS n FROM fingerprints GROUP BY source_id" + ).fetchall() + return {r["source_id"]: r["n"] for r in rows} + + def date_range(self) -> tuple[str | None, str | None]: + """指纹库中最早和最晚的 publish_date。""" + row = self._conn.execute( + "SELECT MIN(publish_date) AS lo, MAX(publish_date) AS hi FROM fingerprints" + ).fetchone() + return (row["lo"], row["hi"]) if row else (None, None) + + # ------------------------------------------------------------------ # + # 写入 + # ------------------------------------------------------------------ # + + def upsert(self, fp: Fingerprint) -> None: + """插入或替换指纹。""" + self._conn.execute( + "INSERT OR REPLACE INTO fingerprints " + "(url_hash, content_hash, simhash_hex, source_id, url, title, " + " publish_date, ingested_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + fp.url_hash, + fp.content_hash, + _to_hex(fp.simhash), + fp.source_id, + fp.url, + fp.title, + fp.publish_date, + fp.ingested_at.isoformat(), + ), + ) + + def delete(self, url_hash: str) -> None: + """删除指定指纹。""" + self._conn.execute("DELETE FROM fingerprints WHERE url_hash = ?", (url_hash,)) + + def clear(self) -> None: + """清空指纹库,主要用于测试。""" + self._conn.execute("DELETE FROM fingerprints") diff --git a/docs/intlnews_usage.html b/docs/intlnews_usage.html new file mode 100644 index 0000000..1b831be --- /dev/null +++ b/docs/intlnews_usage.html @@ -0,0 +1,661 @@ + + + + + +国际财经 Deep Research 平台 — 使用手册 v1.0 + + + + + + +
+
+

🌍 国际财经 Deep Research 平台

+

使用手册 v1.0 · 2026-06-21 · 私有化部署

+
+
+ +
+ + +
+ 📑 目录 +
    +
  1. 项目概述
  2. +
  3. 部署拓扑
  4. +
  5. 环境配置
  6. +
  7. CLI 命令参考
  8. +
  9. M1 — 新闻抓取
  10. +
  11. M2 — 正文提取
  12. +
  13. M3 — 三层去重
  14. +
  15. M4 — 翻译 + 事件抽取
  16. +
  17. M5 — 向量生成
  18. +
  19. M6 — Qdrant 入库与检索
  20. +
  21. M7 — 全链路管道 + 日报
  22. +
  23. M8 — MCP 服务
  24. +
  25. 数据目录结构
  26. +
  27. 定时任务
  28. +
  29. 常见问题
  30. +
+
+ + +

1. 项目概述

+ +

本项目构建面向国际英文财经新闻的私有化 Deep Research 平台。

+ +
+
+ ✅ 核心能力 +
    +
  • 英文财经新闻抓取(Crawl4AI,12 源)
  • +
  • 正文提取(trafilatura)
  • +
  • 全文英译中(DeepSeek LLM)
  • +
  • 投资事件抽取 + 美股代码识别
  • +
  • 双语向量知识库(Qdrant 1024维)
  • +
  • 中文语义检索
  • +
  • MCP 服务(Agent 深度研究)
  • +
  • 每日 AI 摘要日报(HTML)
  • +
+
+
+ ❌ 本项目不是 +
    +
  • 自动交易系统
  • +
  • 股票预测系统
  • +
  • 投资顾问系统
  • +
+
+
+ +
+ 💡 对标项目:A 股 Deep Research(news/),架构模式可复用。 +
+ + +

2. 部署拓扑

+ +
┌──────────────────────────────────────────────────┐
+│           Overseas Server (海外)                   │
+│  M1 Crawl4AI 抓取 → data/raw/                     │
+│  每天 4 次打包 → rsync 推送                        │
+│  SSH: <海外服务器>                               │
+│  路径: /opt/intlgrab                               │
+└────────────────────┬─────────────────────────────┘
+                     │ rsync
+                     ▼
+┌──────────────────────────────────────────────────┐
+│           Domestic Server (国内)                   │
+│  M2 正文提取 → M3 去重 → M4 翻译+事件              │
+│  → M5 向量生成 → M6 Qdrant 入库                    │
+│  → M7 调度 + 日报 → M8 MCP 服务                    │
+│  SSH: <国内服务器>                                │
+│  路径: /home/pi/intlnews                           │
+└──────────────────────────────────────────────────┘
+ + +

3. 环境配置

+ +

3.1 依赖安装

+
cd /home/pi/intlnews
+uv sync
+ +

3.2 配置文件

+ + + + + + +
文件用途注意
.envAPI Key / URL不入 Git,从 .env.example 复制
configs/system.yaml功能参数模型名、阈值、超时、调度
configs/sources.yaml新闻源定义12 个英文财经源
+ +

3.3 必需环境变量

+ +
# DeepSeek(M4 翻译+事件抽取)
+DEEPSEEK_API_KEY=sk-your-key
+DEEPSEEK_BASE_URL=https://api.deepseek.com
+
+# Qwen(备选 LLM)
+QWEN_API_KEY=sk-your-key
+QWEN_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
+
+# DashScope(M5 向量生成)
+DASHSCOPE_API_KEY=sk-your-key
+
+# Qdrant(留空使用本地文件模式)
+QDRANT_URL=http://localhost:6333
+QDRANT_API_KEY=
+ +

3.4 关键配置项

+ +
# configs/system.yaml(关键项)
+crawler.max_memory_mb: 1800       # 内存上限
+dedup.hamming_distance_threshold: 3
+dedup.simhash_window_days: 30
+llm.provider: "deepseek"
+llm.deepseek_model: "deepseek-v4-flash"
+llm.concurrency: 3
+embedding.dashscope_model: "text-embedding-v3"
+embedding.dimension: 1024
+qdrant.collection: "en_finance_news"
+schedule.day_cutoff_hour: 6       # 新闻日切分点
+ + +

4. CLI 命令参考

+ +

所有命令通过 uv run en-news 执行:

+ + + + + + + + + + + + + +
命令Milestone功能
crawlM1抓取英文财经新闻
extractM2正文提取
dedupM3三层去重
translateM4翻译 + 事件抽取
embedM5向量生成
indexM6Qdrant 入库
search <query>M6语义检索
pipelineM7一键全链路 M2→M6
reportM7日报生成
mcp-serverM8启动 MCP 服务
+ +
# 常用选项
+uv run en-news crawl --source forexlive          # 指定源
+uv run en-news search "美联储" --top-k 5          # 指定返回数
+uv run en-news index --recreate                   # 重建 Qdrant
+uv run en-news pipeline --skip-report             # 跳过日报
+ + +

5. M1 — 新闻抓取

+ +

5.1 手动抓取

+
uv run en-news crawl               # 全部源
+uv run en-news crawl -s forexlive  # 指定源
+ +

5.2 12 个英文财经源

+ + + + + + + + + + + + + + + +
源 ID名称类型
reutersReuters综合财经
cnbcCNBC市场新闻
marketwatchMarketWatch市场数据
ftFinancial Times财经深度
yahoo_financeYahoo Finance综合
investingInvesting.com全球市场
seekingalphaSeeking Alpha投资分析
barronsBarrons市场评论
wsjWSJ综合财经
economistThe Economist经济分析
forexliveForexLive外汇新闻
zerohedgeZeroHedge另类财经
+ +

5.3 增量抓取机制

+ +

抓取采用两层增量确保不重复下载和存储:

+ + + + + +
层级位置机制
抓取层_extract_article_urls读取当日 index.jsonl 中已抓取的 url_hash,跳过已存在的 URL,不重复下载
存储层write_index_jsonl追加写入前再次检查 url_hash,已存在则跳过
+ +

同一天内多次执行 crawl,只有新文章才会被下载和存储。

+ +

5.4 产物

+
data/raw/{source_id}/{YYYYMMDD}/
+├── {url_hash}.html     # 原始 HTML
+├── {url_hash}.md       # Crawl4AI Markdown
+└── index.jsonl         # 文章索引
+ +

5.5 反爬策略

+ +

部分新闻源有反爬保护(如 DataDome)。系统支持三种策略,按优先级自动选择:

+ + + + + + +
优先级策略配置说明
1RSS 抓取rss_url通过 Feed 获取文章,完全绕过反爬 ✅ MarketWatch
2Stealthanti_bot_mode: "stealth"隐藏 webdriver 特征
3Headfulanti_bot_mode: "headful"非 headless 浏览器,最像真人
+ +

已验证:MarketWatch RSS 10/10 ✅ · WSJ RSS 20/20 ✅ · ForexLive 17/17 ✅ · ZeroHedge 15/15 ✅ · Barron's RSS 10/10 ✅ · Reuters ⚠️ · SeekingAlpha ⚠️

+ + +

6. M2 — 正文提取

+ +
uv run en-news extract              # 全部源
+uv run en-news extract -s forexlive # 指定源
+ +

技术方案:优先 Crawl4AI Markdown → 回退 trafilatura(英文优化)。最小正文 50 词。

+ +
data/processed/{source_id}/{YYYYMMDD}/
+└── {url_hash}.json     # ProcessedArticle
+ + +

7. M3 — 三层去重

+ +
uv run en-news dedup
+ + + + + + +
层级方法说明
L1URL Hash完全相同 URL
L2内容 HashSHA1[:16] 去标点空白后匹配
L3SimHash字符 3-gram,汉明距离 ≤ 3,30 天窗口
+ +
data/dedup/fingerprints.sqlite3   # SQLite 指纹库
+data/deduped/{YYYYMMDD}/uniques/  # 唯一文章
+ + +

8. M4 — 翻译 + 事件抽取

+ +
uv run en-news translate
+ +
+ ⚙️ 技术规格 + +
+ +

14 种事件类型

+

+ 财报披露 + 并购收购 + 产品发布 + 监管政策 + 宏观经济 + 央行决议 + 行业动态 + 技术突破 + 高管变动 + 诉讼法律 + 市场异动 + 地缘政治 + 大宗商品 + 外汇波动 +

+ +

情绪标签

+

+ 🟢 positive 利好 + ⚪ neutral 中性 + 🔴 negative 利空 +

+ + +

9. M5 — 向量生成

+ +
uv run en-news embed
+ +

技术方案:DashScope text-embedding-v3,1024 维。嵌入文本 = 中文标题 + 事件标签 + 中文正文(4000 字符截断)。

+ +
data/embeddings/{YYYYMMDD}/
+└── {url_hash}.json     # EmbeddingResult (1024 floats)
+ + +

10. M6 — Qdrant 入库与检索

+ +

入库

+
uv run en-news index              # 增量入库
+uv run en-news index --recreate   # 重建 collection
+ +

语义搜索

+
uv run en-news search "美联储利率决议"
+uv run en-news search "伊朗霍尔木兹海峡" --top-k 5
+ +
+ 📦 Qdrant 规格 + +
+ + +

11. M7 — 全链路管道 + 日报

+ +

手动全链路运行(完整流程)

+ +

当需要手工执行完整的数据处理流程时,按顺序执行:

+ +
# 步骤 1(海外): 抓取英文财经新闻
+ssh <海外服务器> "cd /opt/intlgrab && uv run en-news crawl"
+
+# 步骤 2(海外): 打包 raw 数据
+ssh <海外服务器> "cd /opt/intlgrab && bash scripts/overseas_pack.sh"
+
+# 步骤 3(国内): 拉取海外数据
+cd /home/pi/intlnews && bash scripts/domestic_sync.sh
+
+# 步骤 4: 一键全链路 M2→M6 + 日报
+cd /home/pi/intlnews && uv run en-news pipeline
+ +

也可以分步执行(适合调试):

+ +
uv run en-news extract              # M2: 正文提取
+uv run en-news dedup                # M3: 三层去重
+uv run en-news translate            # M4: 翻译 + 事件抽取
+uv run en-news embed                # M5: 向量生成
+uv run en-news index                # M6: Qdrant 入库
+uv run en-news report               # 日报生成
+ +

一键管道(自动)

+
uv run en-news pipeline               # 全链路 M2→M6 + 日报
+uv run en-news pipeline --skip-report  # 跳过日报
+uv run en-news report                  # 仅日报
+ +

管道流程

+
+ M2 extract + + M3 dedup + + M4 translate + + M5 embed + + M6 index + + 📰 日报 +
+ +
+ ⚠️ 降级策略:每步失败记录日志但不阻断后续步骤。 +
+ +

HTML 日报五板块

+
    +
  1. 🤖 AI 摘要 — LLM 根据 important≥4 事件生成要点总结
  2. +
  3. 🔥 重要事件 — 事件表格(情绪/重要度/摘要/链接)
  4. +
  5. 📊 数据总览 — M1→M6 管道统计数字
  6. +
  7. 📈 情绪分布 — 利好/利空/中性比例条
  8. +
  9. 📋 事件类型 TOP 10
  10. +
+ +

本地: data/reports/intl_news_daily_{YYYYMMDD}.html

+

线上: 自动上传到 https://echart.doorcome.cn/research/{YYYYMMDD}/

+

上传配置在 configs/system.yamlreport.upload_host / report.upload_path

+ + +

12. M8 — MCP 服务

+ +

启动

+
uv run en-news mcp-server
+ +

5 个 MCP Tool

+ + + + + + + + +
Tool功能示例调用
search_news语义检索新闻search_news("美联储利率决议")
search_by_stock美股代码检索search_by_stock("AAPL")
search_by_sentiment情绪过滤检索search_by_sentiment("加息", sentiment="negative")
get_today_events当日重要事件get_today_events(importance_min=4)
get_stats系统统计get_stats()
+ +

Claude Code MCP 配置

+
{
+  "mcpServers": {
+    "intl-news": {
+      "command": "uv",
+      "args": ["run", "en-news", "mcp-server"],
+      "cwd": "/home/pi/intlnews"
+    }
+  }
+}
+ + +

13. 数据目录结构

+ +
data/
+├── raw/                         # M1: 原始抓取
+│   └── {source_id}/{YYYYMMDD}/
+├── processed/                   # M2: 正文提取
+│   └── {source_id}/{YYYYMMDD}/
+├── dedup/                       # M3: 指纹库 + 去重结果
+│   ├── fingerprints.sqlite3
+│   └── {YYYYMMDD}/uniques/
+├── events/                      # M4: 翻译+事件
+│   └── {YYYYMMDD}/
+├── embeddings/                  # M5: 向量
+│   └── {YYYYMMDD}/
+├── qdrant_storage/              # M6: Qdrant 本地存储
+└── reports/                     # M7: 日报
+    └── intl_news_daily_{YYYYMMDD}.html
+ + +

14. 定时任务

+ +

时间线

+ + + + + + + + + + + +
时间服务器动作
05:55海外打包 data/raw/
06:30国内拉取 → M2→M6 管道 → 📰 日报
11:55海外打包
12:00国内拉取 → 管道
17:55海外打包
18:00国内拉取 → 管道
21:55海外打包
22:00国内拉取 → 管道 → 📰 日报
+ +

新闻日定义

+

切分点 day_cutoff_hour: 6。当天 06:00 至次日 05:59 属于同一个新闻日。

+ + +

15. 常见问题

+ +

Q: 如何新增新闻源?

+

编辑 configs/sources.yaml

+
- id: "new_source"
+  name: "New Source Name"
+  enabled: true
+  homepage: "https://example.com/finance/"
+  article_url_pattern: "/news/[^/]+/"
+  js_render: false
+  max_articles_per_run: 30
+ +

Q: 翻译质量不好怎么办?

+
    +
  1. 调整 configs/system.yamlllm.temperature(降低更保守)
  2. +
  3. 编辑 prompts/translation_and_extraction.md 优化 Prompt
  4. +
  5. 切换 Provider:llm.provider: "qwen"
  6. +
+ +

Q: Qdrant 检索太慢?

+

本地文件模式 17 条 < 0.01s,足够快。数据量 > 10 万条时切换到 Docker 模式。

+ +

Q: 如何查看日志?

+
tail -f logs/sync.log
+tail -f logs/crawler.log   # 海外
+ +

Q: 数据如何备份?

+
tar czf intlnews_backup_$(date +%Y%m%d).tar.gz data/
+ + +

附录:技术栈

+ + + + + + + + + + + + + + + +
组件技术
语言Python 3.11
包管理uv + pyproject.toml
抓取Crawl4AI + Playwright
正文提取trafilatura
去重SimHash + SQLite
LLMDeepSeek v4-flash(OpenAI SDK)
EmbeddingDashScope text-embedding-v3
向量库Qdrant(本地文件模式)
MCPFastMCP
CLITyper
配置YAML + .env
数据模型Pydantic v2
+ +
+ + + + + diff --git a/docs/intlnews_usage.md b/docs/intlnews_usage.md new file mode 100644 index 0000000..c01d582 --- /dev/null +++ b/docs/intlnews_usage.md @@ -0,0 +1,672 @@ +# 国际财经 Deep Research 平台 — 使用手册 + +> 版本:v1.0 +> 最后更新:2026-06-21 +> 项目路径:国内 `/home/pi/intlnews` / 海外 `/opt/intlgrab` + +--- + +## 目录 + +1. [项目概述](#1-项目概述) +2. [部署拓扑](#2-部署拓扑) +3. [环境配置](#3-环境配置) +4. [CLI 命令参考](#4-cli-命令参考) +5. [M1 — 新闻抓取](#5-m1--新闻抓取) +6. [M2 — 正文提取](#6-m2--正文提取) +7. [M3 — 三层去重](#7-m3--三层去重) +8. [M4 — 翻译 + 事件抽取](#8-m4--翻译--事件抽取) +9. [M5 — 向量生成](#9-m5--向量生成) +10. [M6 — Qdrant 入库与检索](#10-m6--qdrant-入库与检索) +11. [M7 — 全链路管道 + 日报](#11-m7--全链路管道--日报) +12. [M8 — MCP 服务](#12-m8--mcp-服务) +13. [数据目录结构](#13-数据目录结构) +14. [定时任务](#14-定时任务) +15. [常见问题](#15-常见问题) + +--- + +## 1. 项目概述 + +本项目构建面向**国际英文财经新闻**的私有化 Deep Research 平台。 + +核心能力: + +- 英文财经新闻抓取(Crawl4AI,12 个源) +- 正文提取(trafilatura) +- 全文英译中(DeepSeek LLM) +- 投资事件抽取(美股代码识别 + 情绪判断 + 重要度评分) +- 双语向量知识库(Qdrant,1024 维) +- 语义检索(中文自然语言) +- MCP 服务(Claude Code / Cherry Studio Agent 深度研究) +- 每日 AI 摘要日报(HTML) + +**本项目不是**:交易系统 / 股票预测系统 / 投资顾问系统。 + +--- + +## 2. 部署拓扑 + +``` +┌──────────────────────────────────────────────────┐ +│ Overseas Server (海外) │ +│ M1 Crawl4AI 抓取 → data/raw/ │ +│ 每天 4 次打包 → rsync 推送 │ +│ SSH: <海外服务器> │ +│ 路径: /opt/intlgrab │ +└────────────────────┬─────────────────────────────┘ + │ rsync + ▼ +┌──────────────────────────────────────────────────┐ +│ Domestic Server (国内) │ +│ M2 正文提取 → M3 去重 → M4 翻译+事件 │ +│ → M5 向量生成 → M6 Qdrant 入库 │ +│ → M7 调度 + 日报 → M8 MCP 服务 │ +│ SSH: <国内服务器> │ +│ 路径: /home/pi/intlnews │ +└──────────────────────────────────────────────────┘ +``` + +--- + +## 3. 环境配置 + +### 3.1 依赖安装 + +```bash +cd /home/pi/intlnews +uv sync +``` + +### 3.2 配置文件 + +| 文件 | 用途 | 示例 | +|------|------|------| +| `.env` | API Key / URL(不入 Git) | `DEEPSEEK_API_KEY=sk-xxx` | +| `configs/system.yaml` | 功能参数(模型、阈值、超时) | `llm.provider: deepseek` | +| `configs/sources.yaml` | 新闻源定义 | 12 个英文财经源 | + +### 3.3 必需环境变量(`.env`) + +```bash +# DeepSeek(M4 翻译+事件抽取) +DEEPSEEK_API_KEY=sk-your-key +DEEPSEEK_BASE_URL=https://api.deepseek.com + +# Qwen(备选 LLM) +QWEN_API_KEY=sk-your-key +QWEN_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 + +# DashScope(M5 向量生成) +DASHSCOPE_API_KEY=sk-your-key + +# Qdrant(留空使用本地文件模式) +QDRANT_URL=http://localhost:6333 +QDRANT_API_KEY= +``` + +### 3.4 关键配置项(`configs/system.yaml`) + +```yaml +crawler: + max_memory_mb: 1800 # 串行抓取内存上限 + +dedup: + hamming_distance_threshold: 3 # SimHash 汉明距离阈值 + simhash_window_days: 30 # 时间窗口 + +llm: + provider: "deepseek" + deepseek_model: "deepseek-v4-flash" + concurrency: 3 # LLM 并发数 + +embedding: + provider: "dashscope" + dashscope_model: "text-embedding-v3" + dimension: 1024 + +qdrant: + collection: "en_finance_news" + +schedule: + day_cutoff_hour: 6 # 新闻日切分点(06:00) +``` + +--- + +## 4. CLI 命令参考 + +所有命令通过 `uv run en-news` 执行: + +| 命令 | Milestone | 功能 | +|------|-----------|------| +| `crawl` | M1 | 抓取英文财经新闻 | +| `extract` | M2 | 正文提取 | +| `dedup` | M3 | 三层去重 | +| `translate` | M4 | 翻译 + 事件抽取 | +| `embed` | M5 | 向量生成 | +| `index` | M6 | Qdrant 入库 | +| `search ` | M6 | 语义检索 | +| `pipeline` | M7 | 一键全链路 M2→M6 | +| `report` | M7 | 日报生成 | +| `mcp-server` | M8 | 启动 MCP 服务 | + +常用选项: + +```bash +# 指定源 +uv run en-news crawl --source forexlive +uv run en-news extract --source forexlive + +# 指定 top_k +uv run en-news search "美联储利率决议" --top-k 5 + +# 重建 Qdrant collection +uv run en-news index --recreate + +# 全链路跳过日报 +uv run en-news pipeline --skip-report +``` + +--- + +## 5. M1 — 新闻抓取 + +### 5.1 手动抓取 + +```bash +# 抓取所有启用的源 +uv run en-news crawl + +# 只抓取指定源 +uv run en-news crawl -s forexlive +``` + +### 5.2 新闻源列表 + +| 源 ID | 名称 | 类型 | +|-------|------|------| +| reuters | Reuters | 综合财经 | +| cnbc | CNBC | 市场新闻 | +| marketwatch | MarketWatch | 市场数据 | +| ft | Financial Times | 财经深度 | +| yahoo_finance | Yahoo Finance | 综合 | +| investing | Investing.com | 全球市场 | +| seekingalpha | Seeking Alpha | 投资分析 | +| barrons | Barrons | 市场评论 | +| wsj | WSJ | 综合财经 | +| economist | The Economist | 经济分析 | +| forexlive | ForexLive | 外汇新闻 | +| zerohedge | ZeroHedge | 另类财经 | + +### 5.3 反爬策略 + +部分新闻源设有反爬保护(如 DataDome)。系统支持三种策略,按优先级自动选择: + +| 优先级 | 策略 | 配置字段 | 说明 | 适用源 | +|--------|------|---------|------|--------| +| 1 | **RSS 抓取** | `rss_url` | 通过 RSS/Atom Feed 获取文章列表,完全绕过反爬 | MarketWatch ✅ | +| 2 | **Stealth 模式** | `anti_bot_mode: "stealth"` | 隐藏 webdriver 特征(`--disable-blink-features=AutomationControlled`) | Reuters(海外内存不足待验证) | +| 3 | **Headful 模式** | `anti_bot_mode: "headful"` | 非 headless 浏览器,最像真人 | 重度反爬源回退 | + +配置示例(`configs/sources.yaml`): + +```yaml +- id: "marketwatch" + rss_url: "https://feeds.marketwatch.com/marketwatch/topstories" # RSS 优先 + anti_bot_mode: "headful" # RSS 失败时回退 + +- id: "reuters" + anti_bot_mode: "stealth" # 无 RSS,直接 stealth +``` + +**已验证**: + +| 源 | 方式 | 结果 | +|----|------|------| +| MarketWatch | RSS | ✅ 10/10 | +| WSJ | RSS | ✅ 20/20 | +| ForexLive | 标准 headless | ✅ 17/17 | +| ZeroHedge | 标准 + URL 过滤 | ✅ 15/15 | +| Barron's | RSS (Dow Jones) | ✅ 10/10 | +| Reuters | stealth | ⚠️ DataDome | +| SeekingAlpha | stealth | ⚠️ PerimeterX | + +### 5.4 海外定时抓取 + +```cron +# crontab(<海外服务器>)— 每天 4 次 +# 时序: 抓取(60min) → 打包(5min) → 10min后国内拉取 +0 6 * * * cd /opt/intlgrab && bash scripts/overseas_crawl.sh +5 7 * * * cd /opt/intlgrab && bash scripts/overseas_pack.sh +0 12 * * * cd /opt/intlgrab && bash scripts/overseas_crawl.sh +5 13 * * * cd /opt/intlgrab && bash scripts/overseas_pack.sh +0 18 * * * cd /opt/intlgrab && bash scripts/overseas_crawl.sh +5 19 * * * cd /opt/intlgrab && bash scripts/overseas_pack.sh +0 22 * * * cd /opt/intlgrab && bash scripts/overseas_crawl.sh +5 23 * * * cd /opt/intlgrab && bash scripts/overseas_pack.sh +``` + +### 5.4 增量抓取机制 + +抓取采用**两层增量**确保不重复下载和存储: + +| 层级 | 位置 | 机制 | +|------|------|------| +| 抓取层 | `_extract_article_urls` | 读取当日 `index.jsonl` 中已抓取的 url_hash,跳过已存在的 URL,不重复下载 | +| 存储层 | `write_index_jsonl` | 追加写入前再次检查 url_hash,已存在则跳过 | + +同一天内多次执行 `crawl`,只有新文章才会被下载和存储。 + +### 5.5 产物 + +``` +data/raw/{source_id}/{YYYYMMDD}/ +├── {url_hash}.html # 原始 HTML +├── {url_hash}.md # Crawl4AI Markdown +└── index.jsonl # 文章索引 +``` + +--- + +## 6. M2 — 正文提取 + +### 6.1 执行 + +```bash +# 提取所有源 +uv run en-news extract + +# 提取指定源 +uv run en-news extract -s forexlive +``` + +### 6.2 技术方案 + +- 优先使用 Crawl4AI 输出的 Markdown +- 回退 `trafilatura` 英文正文提取 +- 最小正文字数阈值:50 词 + +### 6.3 产物 + +``` +data/processed/{source_id}/{YYYYMMDD}/ +├── {url_hash}.json # ProcessedArticle +└── index.jsonl # 处理索引 +``` + +--- + +## 7. M3 — 三层去重 + +### 7.1 执行 + +```bash +uv run en-news dedup +``` + +### 7.2 去重逻辑 + +| 层级 | 方法 | 说明 | +|------|------|------| +| L1 | URL Hash | 完全相同 URL 直接命中 | +| L2 | 内容 Hash | 标准化后 SHA1[:16] 匹配(去标点/空白) | +| L3 | SimHash | 字符 3-gram,汉明距离 ≤ 3,30 天窗口 | + +### 7.3 产物 + +``` +data/deduped/{YYYYMMDD}/ +├── uniques/{url_hash}.json # 唯一文章 +└── index.json # 去重索引 +data/dedup/fingerprints.sqlite3 # 指纹库 +``` + +--- + +## 8. M4 — 翻译 + 事件抽取 + +### 8.1 执行 + +```bash +uv run en-news translate +``` + +### 8.2 技术方案 + +- **Provider**: DeepSeek v4-flash(默认)/ Qwen 备选 +- **单次调用**:翻译 + 事件抽取合并,节省 token +- **并发**:3 线程(`system.yaml` → `llm.concurrency`) +- **重试**:3 次指数退避(1s → 2s → 4s) + +### 8.3 输出格式 + +```json +{ + "title": "Fed Holds Rates Steady as Markets Rally", + "title_zh": "美联储维持利率不变,市场上涨", + "content_en": "The Federal Reserve held...", + "content_zh": "美联储周三维持利率不变...", + "events": [ + { + "event_type": "央行决议", + "stock_codes": [], + "sentiment": "neutral", + "importance": 5, + "summary_zh": "美联储维持利率不变,市场反弹" + } + ] +} +``` + +### 8.4 14 种事件类型 + +`财报披露` `并购收购` `产品发布` `监管政策` `宏观经济` +`央行决议` `行业动态` `技术突破` `高管变动` `诉讼法律` +`市场异动` `地缘政治` `大宗商品` `外汇波动` `其他` + +### 8.5 产物 + +``` +data/events/{YYYYMMDD}/ +├── {url_hash}.json # EnTranslatedArticle +└── index.json # 事件索引 +``` + +--- + +## 9. M5 — 向量生成 + +### 9.1 执行 + +```bash +uv run en-news embed +``` + +### 9.2 技术方案 + +- **Provider**: DashScope `text-embedding-v3` +- **维度**: 1024 +- **嵌入文本**: `标题: {title_zh}` + `事件: [{sentiment}] {event_type} 重要度{n} {summary_zh}` + `正文: {content_zh[:3000]}` +- **截断**: 4000 字符上限 + +### 9.3 产物 + +``` +data/embeddings/{YYYYMMDD}/ +├── {url_hash}.json # EmbeddingResult(1024 维) +└── index.json # 向量索引 +``` + +--- + +## 10. M6 — Qdrant 入库与检索 + +### 10.1 入库 + +```bash +# 增量入库 +uv run en-news index + +# 重建 collection(清空旧数据) +uv run en-news index --recreate +``` + +### 10.2 语义搜索 + +```bash +# 基本搜索 +uv run en-news search "美联储利率决议" + +# 指定返回条数 +uv run en-news search "伊朗霍尔木兹海峡" --top-k 5 +``` + +### 10.3 技术方案 + +- **模式**: 本地文件(`data/qdrant_storage/`),无需 Docker +- **Collection**: `en_finance_news` +- **距离**: Cosine +- **Payload**: title / title_zh / url / source_id / events / content_zh_preview + +### 10.4 产物 + +``` +data/qdrant_storage/ # Qdrant 本地文件存储 +``` + +--- + +## 11. M7 — 全链路管道 + 日报 + +### 11.1 手动全链路运行(完整流程) + +当需要手工执行完整的数据处理流程时,按顺序执行以下命令: + +```bash +# 步骤 1(海外): 抓取英文财经新闻 +ssh <海外服务器> "cd /opt/intlgrab && uv run en-news crawl" + +# 步骤 2(海外): 打包 raw 数据 +ssh <海外服务器> "cd /opt/intlgrab && bash scripts/overseas_pack.sh" + +# 步骤 3(国内): 拉取海外数据 +cd /home/pi/intlnews && bash scripts/domestic_sync.sh + +# 步骤 4: 一键全链路 M2→M6 + 日报 +cd /home/pi/intlnews && uv run en-news pipeline +``` + +也可以分步执行(适合调试): + +```bash +# 分步模式 +uv run en-news extract # M2: 正文提取 +uv run en-news dedup # M3: 三层去重 +uv run en-news translate # M4: 翻译 + 事件抽取 +uv run en-news embed # M5: 向量生成 +uv run en-news index # M6: Qdrant 入库 +uv run en-news report # 日报生成 +``` + +### 11.2 一键管道(自动) + +```bash +# 全链路 M2→M6 + 日报 +uv run en-news pipeline + +# 跳过日报生成 +uv run en-news pipeline --skip-report + +# 只生成日报 +uv run en-news report +``` + +### 11.2 管道步骤 + +``` +M2 extract → M3 dedup → M4 translate → M5 embed → M6 index → 日报 +``` + +每步失败记录日志但不阻断后续步骤(降级继续)。 + +### 11.3 HTML 日报 + +日报包含五个板块: + +1. 🤖 **AI 摘要** — LLM 根据当日 important≥4 事件生成要点总结 +2. 🔥 **重要事件** — 高重要度事件表格(标题/情绪/重要度/摘要/链接) +3. 📊 **数据总览** — M1→M6 管道统计数据 +4. 📈 **情绪分布** — 利好/利空/中性比例条 + 重要度分布 +5. 📋 **事件类型 TOP 10** + +日报输出:`data/reports/intl_news_daily_{YYYYMMDD}.html`(约 9KB),同时自动上传到 +`https://echart.doorcome.cn/research/{YYYYMMDD}/intl_news_daily_{YYYYMMDD}.html` + +### 11.4 国内定时调度 + +```cron +# crontab(<国内服务器>)— 每天 3 次 +# 全流程:SSH 触发海外打包 → 下载 → 管道串行 M2→M6 → 日报 +0 7 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh +0 12 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh +0 18 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh +``` + +`domestic_full.sh` 统一完成:远程打包 → 同步 → 管道 → 日报。 + +--- + +## 12. M8 — MCP 服务 + +### 12.1 启动 + +```bash +uv run en-news mcp-server +``` + +### 12.2 可用 Tool + +| Tool | 功能 | 示例 | +|------|------|------| +| `search_news` | 语义检索新闻 | `search_news("美联储利率决议")` | +| `search_by_stock` | 美股代码检索 | `search_by_stock("AAPL")` | +| `search_by_sentiment` | 按情绪检索 | `search_by_sentiment("加息", sentiment="negative")` | +| `get_today_events` | 当日重要事件 | `get_today_events(importance_min=4)` | +| `get_stats` | 系统统计 | `get_stats()` | + +### 12.3 Claude Code 配置 + +在 Claude Code 的 MCP 配置中添加: + +```json +{ + "mcpServers": { + "intl-news": { + "command": "uv", + "args": ["run", "en-news", "mcp-server"], + "cwd": "/home/pi/intlnews" + } + } +} +``` + +--- + +## 13. 数据目录结构 + +``` +data/ +├── raw/ # M1: 原始抓取 +│ └── {source_id}/{YYYYMMDD}/ +│ ├── {url_hash}.html +│ ├── {url_hash}.md +│ └── index.jsonl +├── processed/ # M2: 正文提取 +│ └── {source_id}/{YYYYMMDD}/ +│ └── {url_hash}.json +├── dedup/ # M3: 指纹库 +│ ├── fingerprints.sqlite3 +│ └── {YYYYMMDD}/ +│ ├── uniques/{url_hash}.json +│ └── index.json +├── events/ # M4: 翻译+事件 +│ └── {YYYYMMDD}/ +│ └── {url_hash}.json +├── embeddings/ # M5: 向量 +│ └── {YYYYMMDD}/ +│ └── {url_hash}.json +├── qdrant_storage/ # M6: Qdrant 本地存储 +└── reports/ # M7: 日报 + └── intl_news_daily_{YYYYMMDD}.html +``` + +--- + +## 14. 定时任务 + +### 14.1 时间线 + +``` +海外 国内 +───────────────────────────── ───────────────────────── +06:00 crawl (≈60min) 07:00 全流程(打包→下载→管道→日报) +12:00 crawl (≈60min) 12:00 全流程 +18:00 crawl (≈60min) 18:00 全流程 +22:00 crawl (≈60min) (夜间 crawl 结果次日 07:00 处理) +``` + +国内 `domestic_full.sh` 流程:`SSH触发海外打包 → 下载 → M2→M3→M4→M5→M6 → 日报`(串行)。 + +### 14.2 新闻日定义 + +- 切分点:`day_cutoff_hour: 6`(凌晨 06:00) +- 当天 06:00 至次日 05:59 属于同一个新闻日 +- 例如:2026-06-19 04:00 → 新闻日 "20260618" + +--- + +## 15. 常见问题 + +### Q: 如何新增新闻源? + +编辑 `configs/sources.yaml`,添加源配置: + +```yaml +- id: "new_source" + name: "New Source Name" + enabled: true + homepage: "https://example.com/finance/" + article_url_pattern: "/news/[^/]+/" + js_render: false + max_articles_per_run: 30 +``` + +### Q: 翻译质量不好怎么办? + +1. 调整 `configs/system.yaml` 中 `llm.temperature`(降低更保守) +2. 编辑 `prompts/translation_and_extraction.md` 优化 Prompt +3. 切换 Provider:`llm.provider: "qwen"` + +### Q: Qdrant 检索太慢? + +- 本地文件模式已足够快(17 条 < 0.01s) +- 数据量 > 10 万条时建议切换到 Docker 模式 +- 设置 `QDRANT_URL=http://your-server:6333` + +### Q: 如何查看日志? + +```bash +tail -f logs/sync.log # 同步日志(国内) +tail -f logs/pipeline.log # 管道日志(国内) +tail -f logs/crawl.log # 抓取日志(海外) +tail -f logs/pack.log # 打包日志(海外) +``` + +日志自动清理:每周日凌晨 3 点删除 14 天前的 `.log` 文件(`scripts/cleanup_logs.sh`)。 + +### Q: 数据如何备份? + +```bash +# 备份整个 data 目录 +tar czf intlnews_backup_$(date +%Y%m%d).tar.gz data/ +``` + +--- + +## 附录:技术栈 + +| 组件 | 技术 | +|------|------| +| 语言 | Python 3.11 | +| 包管理 | uv + pyproject.toml | +| 抓取 | Crawl4AI + Playwright | +| 正文提取 | trafilatura | +| 去重 | SimHash + SQLite | +| LLM | DeepSeek v4-flash(OpenAI SDK) | +| Embedding | DashScope text-embedding-v3 | +| 向量库 | Qdrant(本地文件模式) | +| MCP | FastMCP | +| CLI | Typer | +| 配置 | YAML + .env | +| 数据模型 | Pydantic v2 | diff --git a/embedding/__init__.py b/embedding/__init__.py new file mode 100644 index 0000000..2eac7ff --- /dev/null +++ b/embedding/__init__.py @@ -0,0 +1,35 @@ +"""向量生成模块 (M5)。 + +公共 API: + - load_embedding_config / make_embedding_client + - compose_text / embed_article / embed_articles + - embed_all_events(批量管道) + - EmbeddingResult / EmbeddingConfig / EmbeddingError +""" + +from embedding.client import ( + EmbeddingConfig, + embed_batch, + load_embedding_config, + make_embedding_client, +) +from embedding.embedder import compose_text, embed_article, embed_articles +from embedding.models import EmbeddingError, EmbeddingResult +from embedding.pipeline import embed_all_events + +__all__ = [ + # 客户端 + "EmbeddingConfig", + "embed_batch", + "load_embedding_config", + "make_embedding_client", + # 嵌入 + "compose_text", + "embed_article", + "embed_articles", + # 批量管道 + "embed_all_events", + # 模型 + "EmbeddingError", + "EmbeddingResult", +] diff --git a/embedding/client.py b/embedding/client.py new file mode 100644 index 0000000..1b70225 --- /dev/null +++ b/embedding/client.py @@ -0,0 +1,198 @@ +"""DashScope Embedding 客户端。 + +通过 OpenAI 兼容接口调用阿里百炼 text-embedding-v3: + base_url: https://dashscope.aliyuncs.com/compatible-mode/v1 + model: text-embedding-v3(1024 维) + +配置来源: + - .env → DASHSCOPE_API_KEY / QWEN_BASE_URL(OpenAI 兼容端点) + - configs/system.yaml → embedding 段(model / dimension / batch_size / timeout) +""" + +import logging +import os +import time +from dataclasses import dataclass +from pathlib import Path + +import yaml +from openai import OpenAI + +from embedding.models import EmbeddingError + +logger = logging.getLogger(__name__) + +# 默认值 +DASHSCOPE_DEFAULT_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1" +DASHSCOPE_DEFAULT_MODEL = "text-embedding-v3" +DASHSCOPE_DEFAULT_DIM = 1024 +DASHSCOPE_BATCH_LIMIT = 10 # 百炼实测单批上限 + +# 重试 +DEFAULT_MAX_ATTEMPTS = 3 +RETRY_BASE_WAIT_SEC = 1.0 +RETRY_MAX_WAIT_SEC = 8.0 + + +def _load_embedding_config() -> dict: + """从 system.yaml 加载 embedding 段配置。""" + config_path = Path("configs/system.yaml") + if config_path.exists(): + try: + with open(config_path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + return raw.get("embedding", {}) + except Exception: + logger.warning("加载 embedding 配置失败") + return {} + + +@dataclass +class EmbeddingConfig: + """Embedding 调用配置。""" + + provider: str = "dashscope" + model: str = DASHSCOPE_DEFAULT_MODEL + api_key: str = "" + base_url: str = DASHSCOPE_DEFAULT_BASE + dimension: int = DASHSCOPE_DEFAULT_DIM + batch_size: int = DASHSCOPE_BATCH_LIMIT + timeout_sec: float = 30.0 + max_attempts: int = DEFAULT_MAX_ATTEMPTS + + def __post_init__(self) -> None: + if not self.api_key: + raise EmbeddingError("DASHSCOPE_API_KEY 未配置,请检查 .env") + + +def load_embedding_config( + *, + model: str | None = None, +) -> EmbeddingConfig: + """根据配置构造 EmbeddingConfig。 + + 优先级: system.yaml > .env 默认值 > 硬编码默认值 + """ + sys_cfg = _load_embedding_config() + + # API key: 从环境变量读取 + api_key = os.environ.get("DASHSCOPE_API_KEY", "") + if not api_key: + api_key = os.environ.get("QWEN_API_KEY", "") + + # Base URL: 优先用 QWEN_BASE_URL(OpenAI 兼容), + # DASHSCOPE_BASE_URL 通常是旧版非兼容端点,不作为默认 + base_url = ( + os.environ.get("QWEN_BASE_URL") + or os.environ.get("DASHSCOPE_EMBEDDING_BASE_URL") + or DASHSCOPE_DEFAULT_BASE + ) + + m = model or sys_cfg.get("dashscope_model", DASHSCOPE_DEFAULT_MODEL) + dimension = int(sys_cfg.get("dimension", DASHSCOPE_DEFAULT_DIM)) + batch_size = min(int(sys_cfg.get("batch_size", DASHSCOPE_BATCH_LIMIT)), DASHSCOPE_BATCH_LIMIT) + timeout = float(sys_cfg.get("timeout_sec", 30.0)) + + if not api_key: + raise EmbeddingError("DASHSCOPE_API_KEY 未配置,请检查 .env") + + return EmbeddingConfig( + provider="dashscope", + model=m, + api_key=api_key, + base_url=base_url, + dimension=dimension, + batch_size=batch_size, + timeout_sec=timeout, + ) + + +def make_embedding_client(config: EmbeddingConfig) -> OpenAI: + """构造同步 OpenAI 客户端(指向 DashScope 兼容端点)。""" + logger.info( + "初始化 Embedding 客户端: provider=%s model=%s base_url=%s dim=%d", + config.provider, config.model, config.base_url, config.dimension, + ) + return OpenAI( + api_key=config.api_key, + base_url=config.base_url, + timeout=config.timeout_sec, + ) + + +def _chunked(items: list[str], size: int) -> list[list[str]]: + """把列表按 size 分块。""" + return [items[i : i + size] for i in range(0, len(items), size)] + + +def embed_batch( + client: OpenAI, + config: EmbeddingConfig, + texts: list[str], +) -> list[list[float]]: + """批量嵌入,自动分块+重试。 + + Args: + client: OpenAI 客户端 + config: Embedding 配置 + texts: 待嵌入文本列表 + + Returns: + 与 texts 等长的向量列表,每个为 1024 维 float 列表 + """ + if not texts: + return [] + + all_results: list[list[float]] = [] + chunks = _chunked(texts, config.batch_size) + + for chunk_idx, chunk in enumerate(chunks): + result = _call_with_retry(client, config, chunk, chunk_idx, len(chunks)) + all_results.extend(result) + + return all_results + + +def _call_with_retry( + client: OpenAI, + config: EmbeddingConfig, + batch: list[str], + chunk_idx: int, + total_chunks: int, +) -> list[list[float]]: + """单批嵌入调用,带指数退避重试。""" + last_err: Exception | None = None + + for attempt in range(1, config.max_attempts + 1): + try: + resp = client.embeddings.create(model=config.model, input=batch) + vectors = [d.embedding for d in resp.data] + + # 维度校验 + if vectors and len(vectors[0]) != config.dimension: + logger.warning( + "实际维度 %d 与预期 %d 不一致", + len(vectors[0]), config.dimension, + ) + + logger.debug( + "Embedding chunk %d/%d 完成(%d 条,attempt %d)", + chunk_idx + 1, total_chunks, len(batch), attempt, + ) + return vectors + + except Exception as e: + last_err = e + logger.warning( + "DashScope embed 失败 chunk %d/%d 尝试 %d/%d: %s: %s", + chunk_idx + 1, total_chunks, attempt, config.max_attempts, + type(e).__name__, e, + ) + if attempt < config.max_attempts: + wait = min(RETRY_BASE_WAIT_SEC * (2 ** (attempt - 1)), RETRY_MAX_WAIT_SEC) + time.sleep(wait) + + raise EmbeddingError( + f"DashScope embed 放弃({config.max_attempts} 次): {last_err}", + attempts=config.max_attempts, + ) diff --git a/embedding/embedder.py b/embedding/embedder.py new file mode 100644 index 0000000..bd4f97d --- /dev/null +++ b/embedding/embedder.py @@ -0,0 +1,141 @@ +"""文本组装与向量生成。 + +核心逻辑: + 1. compose_text: 从 EnTranslatedArticle 拼接中文文本供嵌入 + 2. embed_article: 单篇文章嵌入 + 3. embed_articles: 批量嵌入(自动分块) +""" + +import logging + +from openai import OpenAI + +from embedding.client import EmbeddingConfig, embed_batch +from embedding.models import EmbeddingResult +from llm.models import EnTranslatedArticle + +logger = logging.getLogger(__name__) + +# 嵌入文本最大字符数(DashScope text-embedding-v3 支持 8192 token,保守取 4000 字符) +MAX_EMBED_CHARS = 4000 + + +def compose_text(article: EnTranslatedArticle, max_chars: int = MAX_EMBED_CHARS) -> str: + """把 EnTranslatedArticle 组装成单段嵌入文本。 + + 拼接策略: + - 中文标题(最高信号) + - 事件摘要(语义浓缩) + - 中文正文(截断) + + Args: + article: M4 输出的双语文章 + max_chars: 整段最大字符数 + + Returns: + 拼接后的中文嵌入文本 + """ + parts: list[str] = [] + + # 1. 标题(中文) + if article.title_zh: + parts.append(f"标题: {article.title_zh}") + + # 2. 事件标签(语义浓缩) + for ev in article.events: + event_parts = [ + f"[{ev.sentiment.value}]", + f"{ev.event_type}", + f"重要度{ev.importance}", + ] + if ev.stock_codes: + event_parts.append("代码:" + ",".join(ev.stock_codes[:5])) + if ev.summary_zh: + event_parts.append(ev.summary_zh) + parts.append("事件: " + " ".join(event_parts)) + + # 3. 中文正文 + if article.content_zh: + body = article.content_zh + parts.append(f"正文: {body}") + + text = "\n".join(parts) + + # 截断保护 + if len(text) > max_chars: + logger.debug("嵌入文本超长 %d → %d", len(text), max_chars) + text = text[:max_chars] + + return text + + +def embed_article( + client: OpenAI, + config: EmbeddingConfig, + article: EnTranslatedArticle, +) -> EmbeddingResult: + """单篇文章嵌入。 + + Args: + client: OpenAI 客户端 + config: Embedding 配置 + article: M4 输出的双语文章 + + Returns: + EmbeddingResult 含向量 + 元信息 + """ + text = compose_text(article) + + vectors = embed_batch(client, config, [text]) + if not vectors: + raise ValueError(f"嵌入返回空结果: {article.url_hash}") + + return EmbeddingResult( + url_hash=article.url_hash, + source_id=article.source_id, + vector=vectors[0], + dimension=len(vectors[0]), + embedded_text=text, + provider=config.provider, + model=config.model, + ) + + +def embed_articles( + client: OpenAI, + config: EmbeddingConfig, + articles: list[EnTranslatedArticle], +) -> list[EmbeddingResult]: + """批量嵌入多篇文章(自动分块+重试)。 + + Args: + client: OpenAI 客户端 + config: Embedding 配置 + articles: M4 输出的双语文章列表 + + Returns: + EmbeddingResult 列表 + """ + if not articles: + return [] + + # 组装所有嵌入文本 + texts = [compose_text(a) for a in articles] + + # 批量嵌入 + vectors = embed_batch(client, config, texts) + + # 组装结果 + results: list[EmbeddingResult] = [] + for article, vec in zip(articles, vectors): + results.append(EmbeddingResult( + url_hash=article.url_hash, + source_id=article.source_id, + vector=vec, + dimension=len(vec), + embedded_text=compose_text(article), + provider=config.provider, + model=config.model, + )) + + return results diff --git a/embedding/models.py b/embedding/models.py new file mode 100644 index 0000000..ce75901 --- /dev/null +++ b/embedding/models.py @@ -0,0 +1,34 @@ +"""Embedding 向量生成数据模型 (M5)。""" + +from datetime import datetime + +from pydantic import BaseModel, Field + + +class EmbeddingResult(BaseModel): + """单篇文章的嵌入向量结果,M5 最终落盘格式。""" + + # 来源标识 + url_hash: str + source_id: str + + # 嵌入向量(1024 维 float 列表) + vector: list[float] = Field(..., description="1024 维浮点向量") + dimension: int = 1024 + + # 嵌入文本(用于检索时调试/可视化) + embedded_text: str = Field(default="", description="拼接后送入 embedder 的文本") + + # 调用元信息 + provider: str = "dashscope" + model: str = "" + embedded_at: datetime = Field(default_factory=datetime.now) + + +class EmbeddingError(Exception): + """Embedding 调用失败。""" + + def __init__(self, reason: str, *, attempts: int = 0) -> None: + super().__init__(reason) + self.reason = reason + self.attempts = attempts diff --git a/embedding/pipeline.py b/embedding/pipeline.py new file mode 100644 index 0000000..b944a6b --- /dev/null +++ b/embedding/pipeline.py @@ -0,0 +1,179 @@ +"""批量向量生成管道。 + +输入: data/events/{YYYYMMDD}/{url_hash}.json(M4 翻译+事件输出) +输出: data/embeddings/{YYYYMMDD}/{url_hash}.json +""" + +import json +import logging +from datetime import datetime +from pathlib import Path + +from crawler.utils import get_news_day +from embedding.client import ( + EmbeddingConfig, + load_embedding_config, + make_embedding_client, +) +from embedding.embedder import embed_articles +from embedding.models import EmbeddingError +from llm.models import EnTranslatedArticle + +logger = logging.getLogger(__name__) + + +def _load_event_articles(date_str: str) -> list[EnTranslatedArticle]: + """加载指定日期的翻译+事件文章。 + + Args: + date_str: 日期 YYYYMMDD + + Returns: + EnTranslatedArticle 列表 + """ + base_dir = Path(f"data/events/{date_str}") + if not base_dir.exists(): + return [] + + articles: list[EnTranslatedArticle] = [] + for json_file in sorted(base_dir.glob("*.json")): + if json_file.name == "index.json": + continue + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + articles.append(EnTranslatedArticle(**data)) + except (json.JSONDecodeError, Exception) as e: + logger.warning("解析事件文章失败 %s: %s", json_file, e) + + return articles + + +def embed_all_events( + date_str: str | None = None, + *, + model: str | None = None, +) -> dict: + """对所有 M4 输出的文章执行向量化。 + + Args: + date_str: 日期 YYYYMMDD,默认当前新闻日 + model: Embedding 模型名,默认从 system.yaml 读取 + + Returns: + 统计 dict + """ + if date_str is None: + date_str = get_news_day() + + logger.info("══════ 开始向量生成,日期: %s ══════", date_str) + + # 加载文章 + articles = _load_event_articles(date_str) + if not articles: + logger.warning("事件目录无文章: data/events/%s/", date_str) + return {"date": date_str, "total": 0, "success": 0, "failed": 0, "elapsed_sec": 0} + + # 初始化 Embedding 客户端 + config = load_embedding_config(model=model) + client = make_embedding_client(config) + + # 输出目录 + out_dir = Path(f"data/embeddings/{date_str}") + out_dir.mkdir(parents=True, exist_ok=True) + + # 增量:跳过已向量化的文章 + new_articles = [] + skipped = 0 + for a in articles: + if (out_dir / f"{a.url_hash}.json").exists(): + skipped += 1 + else: + new_articles.append(a) + if skipped > 0: + logger.info("增量跳过 %d 篇已向量化,剩余 %d 篇待处理", skipped, len(new_articles)) + articles = new_articles + + start_time = datetime.now() + success = 0 + failed = 0 + + # 批量嵌入(按 batch_size 分块,每批输出进度) + batch_size = config.batch_size + total = len(articles) + logger.info("开始向量化 %d 篇文章(batch_size=%d, model=%s)", + total, batch_size, config.model) + + for batch_start in range(0, total, batch_size): + batch_end = min(batch_start + batch_size, total) + batch = articles[batch_start:batch_end] + + try: + results = embed_articles(client, config, batch) + for result in results: + out_file = out_dir / f"{result.url_hash}.json" + out_file.write_text( + result.model_dump_json(indent=2, ensure_ascii=False), + encoding="utf-8", + ) + success += 1 + + logger.info(" [%d/%d] ✅ %d 篇 → %d 维向量", + batch_end, total, len(results), config.dimension) + + except EmbeddingError as e: + failed += len(batch) + logger.error("批量嵌入失败 [%d-%d]: %s", batch_start, batch_end, e.reason) + except Exception as e: + failed += len(batch) + logger.exception("批量嵌入异常 [%d-%d]: %s", batch_start, batch_end, e) + + elapsed = (datetime.now() - start_time).total_seconds() + + # 写入索引 + _write_embedding_index(date_str, success, failed, elapsed, config) + + logger.info( + "══════ 向量生成完成: 成功 %d / 失败 %d / 总计 %d,耗时 %.1f 秒 ══════", + success, failed, len(articles), elapsed, + ) + + return { + "date": date_str, + "total": len(articles), + "success": success, + "failed": failed, + "elapsed_sec": elapsed, + "provider": config.provider, + "model": config.model, + "dimension": config.dimension, + } + + +def _write_embedding_index( + date_str: str, + success: int, + failed: int, + elapsed_sec: float, + config: EmbeddingConfig, +) -> None: + """写入向量索引文件。""" + out_dir = Path(f"data/embeddings/{date_str}") + out_dir.mkdir(parents=True, exist_ok=True) + + index_data = { + "date": date_str, + "success": success, + "failed": failed, + "elapsed_sec": round(elapsed_sec, 1), + "provider": config.provider, + "model": config.model, + "dimension": config.dimension, + "generated_at": datetime.now().isoformat(), + } + + index_path = out_dir / "index.json" + index_path.write_text( + json.dumps(index_data, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + logger.info("向量索引已写入: %s", index_path) diff --git a/english-news-plan.md b/english-news-plan.md new file mode 100644 index 0000000..750fe5b --- /dev/null +++ b/english-news-plan.md @@ -0,0 +1,477 @@ +# English Financial News — 项目开发计划 + +> 国际财经新闻抓取与深度研究平台 +> +> 对标 `news/` (A 股 Deep Research),复用其架构模式,聚焦英文财经源。 + +--- + +## 一、项目定位 + +构建面向国际财经新闻的私有化 Deep Research 平台。 + +核心能力: +- 英文财经新闻抓取(Crawl4AI) +- 新闻联播数据接入(api.doorcome.cn) +- 全文英译中(LLM) +- 投资事件抽取(LLM) +- 双语向量知识库(Qdrant) +- MCP 服务 + Cherry Studio / Claude Code Agent 深度研究 +- 每日 AI 摘要日报(含新闻联播投资相关解读) + +本项目不是: +- 交易系统 +- 预测系统 +- 投资顾问 + +--- + +## 二、部署拓扑 + +``` +┌──────────────────────────────────────────────────────────────┐ +│ Overseas Server (国外) │ +│ M1 Crawl4AI 抓取 → data/raw/{source}/{YYYYMMDD}/ │ +│ ↓ │ +│ rsync 增量推送 (定时) │ +└──────────────────────────┬───────────────────────────────────┘ + │ SSH / rsync + ▼ +┌──────────────────────────────────────────────────────────────┐ +│ Domestic Server (国内) │ +│ M2 正文提取 → M3 去重 → M4 翻译+事件抽取(LLM) │ +│ → M5 向量生成 → M6 Qdrant 入库 │ +│ → M7 定时调度 → M8 MCP 服务 │ +│ → 日报生成 (AI 摘要 + 重要事件 + 新闻联播投资解读) │ +│ │ +│ ┌── 新闻联播数据源 (独立 API) ──┐ │ +│ │ GET api.doorcome.cn/api/xwlbFine/ │ +│ │ → LLM 筛选投资相关 → 重要性评分 → AI 解读 │ +│ └──────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────┘ +``` + +同步机制: +- 海外服务器每天定时 `rsync -avz` 推送 `data/raw/` 到国内 +- 国内定时任务先拉取同步,再执行 M2→M8 管道 +- rsync 天然增量(只传新文件),带宽高效 + +--- + +## 三、Milestone 分解 + +### M0 — 项目骨架 + +- `pyproject.toml`(Python 3.11 + uv) +- `.env.example` +- `configs/sources.yaml`(英文财经源配置) +- `CLAUDE.md` +- 目录结构初始化 +- CLI 入口:`en-news` + +### M1 — 新闻抓取(海外服务器) + +**技术选型**:Crawl4AI(异步 + Playwright JS 渲染),与 A 股项目一致。 + +**英文财经源**(初始 12 个): + +| 源 ID | 名称 | 类型 | +|-------|------|------| +| reuters | Reuters | 综合财经 | +| cnbc | CNBC | 市场新闻 | +| marketwatch | MarketWatch | 市场数据 | +| ft | Financial Times | 财经深度 | +| yahoo_finance | Yahoo Finance | 综合 | +| investing | Investing.com | 全球市场 | +| seekingalpha | Seeking Alpha | 投资分析 | +| barrons | Barrons | 市场评论 | +| wsj | WSJ | 综合财经 | +| economist | The Economist | 经济分析 | +| forexlive | ForexLive | 外汇新闻 | +| zerohedge | ZeroHedge | 另类财经 | + +每个源配置字段: +```yaml +- id: "reuters" + name: "Reuters" + enabled: true + homepage: "https://www.reuters.com/business/" + article_url_pattern: "/[^/]+/[^/]+/" + js_render: true + max_articles_per_run: 30 +``` + +**产物**:`data/raw/{source_id}/{YYYYMMDD}/{url_hash}.html` + `index.jsonl` + +**同步守护进程**: +```bash +# 海外 crontab:每天 05:00 UTC 推送 +0 5 * * * rsync -avz --ignore-existing /app/data/raw/ user@domestic:/app/data/raw/ +``` + +### M2 — 正文提取(国内服务器) + +**输入**:`data/raw/{source_id}/{date}/`(rsync 同步后) + +**提取方案**: +- 优先使用 Crawl4AI 输出的 Markdown(`{url_hash}.md`) +- 对未生成 Markdown 的源,用 `trafilatura`(英文正文提取,类 GNE 但针对英文优化) +- 输出标准化 `Article` 模型(`title` / `content` / `publish_time` / `author` / `source_id`) + +**产物**:`data/processed/{source_id}/{YYYYMMDD}/{url_hash}.json` + +### M3 — 去重 + +与 A 股项目相同的三层去重: +1. URL Hash(精确) +2. 内容 Hash(SHA256 前 64 字符) +3. SimHash 模糊(汉明距离 ≤ 3,30 天窗口) + +**产物**:`data/deduped/{YYYYMMDD}/uniques/{url_hash}.json` + +### M4 — 翻译 + 投资事件抽取(LLM) + +**单次 LLM 调用完成两件事**(节省 token): + +Prompt 设计: +``` +输入:英文财经新闻全文 +输出 JSON: +{ + "translation_zh": "中文全文翻译", + "events": [ + { + "event_type": "并购/财报/政策/行业/...", + "stock_codes": ["AAPL", "TSLA"], + "sentiment": "positive/negative/neutral", + "importance": 1-5, + "summary_zh": "事件中文摘要" + } + ] +} +``` + +**LLM Provider**:DeepSeek(默认)/ Qwen 备选 + +**产物**:`data/events/{YYYYMMDD}/{url_hash}.json` +```json +{ + "title": "Apple Reports Record Q2 Earnings", + "title_zh": "苹果公布创纪录第二季度财报", + "content_en": "...", + "content_zh": "...", + "events": [...] +} +``` + +### M5 — 向量生成 + +- Embedding Provider:DashScope `text-embedding-v3`(1024 维) +- 对中文翻译内容向量化(`content_zh` + `title_zh` + 事件摘要拼接) +- 也可选择英文原文向量化,支持双语检索 + +**产物**:`data/embeddings/{YYYYMMDD}/{url_hash}.json` + +### M6 — Qdrant 入库与检索 + +与 A 股项目相同: +- 本地文件模式(默认)或 Docker Server 模式 +- Collection:`en_finance_news` +- Payload:`source_id` / `title` / `title_zh` / `url` / `publish_time` / `events` / `sentiment` / `importance` +- Vector:从 M5 产物加载 + +```python +uv run en-news search "Fed interest rate decision" +``` + +### M7 — 新闻联播数据源(独立模块) + +**数据获取**: + +``` +GET https://api.doorcome.cn/api/xwlbFine/?start_date=YYYY-MM-DD&end_date=YYYY-MM-DD +``` + +**响应结构**: +```json +{ + "status": "success", + "data": { + "news": [ + { + "news_days": "2026-06-20", + "daily_sub_id": 1, + "news_title": "牢记总书记嘱托 各地在文脉赓序中推动城市高质量发展", + "news_improve": "历史文化是城市的灵魂...(全文约3000字)" + } + ] + } +} +``` + +每天约 12-15 条新闻,每条含完整文字稿。 + +**处理流程**: + +1. **获取** — 日报生成前拉取昨日新闻联播数据。例如 6/21 早上执行日报 → 拉取 6/20 的新闻联播(前一日 19:00 播出,次日凌晨 API 已就绪) +2. **投资相关性筛选(LLM)** — 用 DeepSeek 逐条判断是否与投资相关 + - 经济数据、产业政策、贸易谈判 → **高相关**(importance 4-5) + - 科技创新、区域发展、基建项目 → **中相关**(importance 2-3) + - 文化、民生、体育、天气 → **非相关**(跳过) +3. **AI 解读** — 对投资相关条目生成 2-3 句投资视角解读: + - 对哪些行业/板块有影响 + - 政策信号含义 + - 市场可能的反应 +4. **纳入日报** — 单独一节展示,区别于英文财经新闻 + +**产物**:`data/xwlb/{YYYYMMDD}/index.jsonl` +```json +{ + "news_days": "2026-06-20", + "daily_sub_id": 5, + "news_title": "中欧班列统一品牌十周年", + "is_investment_related": true, + "importance": 4, + "category": "贸易/物流", + "ai_interpretation": "中欧班列十年数据将直接反映一带一路贸易活跃度,利好物流、港口、跨境贸易板块..." +} +``` + +### M8 — 定时调度 + +- 国内 crontab 或 APScheduler 守护进程 +- 日期规则:T 日早上生成的日报覆盖 T-1 日数据(新闻 + 新闻联播) +- 流程: + ``` + 06:00 等待海外 rsync 完成(检查哨兵文件) + 06:30 M2→M3→M4→M5→M6 全链路(处理 T-1 日英文新闻) + 07:00 拉取 T-1 日新闻联播 → LLM 筛选解读 → 日报生成 + 12:00 增量(只处理当日新增) + 18:00 增量 + 22:00 增量 + 日报 + ``` + +### M9 — MCP 服务 + +暴露 5 个 MCP 工具给 Cherry Studio / Claude Code: + +| 工具 | 功能 | +|------|------| +| `search_en_news` | 通用语义检索(中文查询 → 中文翻译向量匹配) | +| `search_by_company` | 按公司名/股票代码检索 | +| `search_by_sentiment` | 按情绪检索+统计 | +| `search_by_event_type` | 按事件类型检索 | +| `get_daily_report` | 获取最新日报 | + +--- + +## 四、数据模型 + +```python +class EnArticle(BaseModel): + source_id: str + url: str + url_hash: str + title: str # 英文原标题 + title_zh: str # 中文翻译标题 + content_en: str # 英文原文 + content_zh: str # 中文全文翻译 + author: str + publish_time: datetime + source_name: str + word_count: int + word_count_zh: int + +class EnExtractedEvent(BaseModel): + article_url_hash: str + event_type: str # 并购/财报/政策/行业/市场/... + stock_codes: list[str] # 涉及美股代码 + sentiment: str # positive/negative/neutral + importance: int # 1-5 + summary_zh: str # 中文摘要 + summary_en: str # 英文摘要 + +class XwlbItem(BaseModel): + """新闻联播条目(经 LLM 筛选和解读)。""" + news_days: str # 播出日期 "2026-06-20" + daily_sub_id: int # 当日序号 (1-15) + news_title: str # 标题 + news_improve: str # 完整文字稿 + is_investment_related: bool # 是否投资相关 + importance: int # 投资重要性 1-5 (非投资为 0) + category: str # 投资类别 (贸易/产业政策/科技/基建/金融/...) + ai_interpretation: str # AI 投资解读 2-3 句 +``` + +--- + +## 五、日报设计 + +命令:`uv run en-news pipeline --once --report` + +执行顺序:新闻联播拉取 → LLM 筛选解读 → 日报 HTML 生成 + +六段式结构: + +1. **AI 摘要**(24h 国际财经 + 昨日新闻联播投资相关,≤500 字,最后一条不截断) + +2. **重要事件:国际新闻**(24h 英文源,importance ≥ 4 逐级回退,最多 20 篇) + +3. **📺 新闻联播投资解读**(昨日播出,投资相关条目 + AI 解读) + - 展示格式:标题 | 投资相关性类别 | 重要度 | AI 解读(2-3 句) + - 非投资相关条目不显示 + - 示例: + ``` + 📺 中欧班列统一品牌十周年 [贸易/物流] ⭐⭐⭐⭐ + AI解读: 中欧班列十年数据将直接反映一带一路贸易活跃度。 + 关注物流、港口、跨境贸易板块。政策利好中国外运、中远海控等。 + ``` + +4. **市场情绪分布**(利好/利空/中性,24h 国际新闻 + 新闻联播合并统计) + +5. **数据总览**(M1→M6 管道统计 + 各源抓取量 6 列网格 + 重要度分布 + 事件类型分布 + 新闻联播条目数) + +6. **完整新闻联播条目列表**(当天的全部条目,标题+简要,含非投资类,供快速浏览) + +--- + +## 六、配置规范 + +### configs/sources.yaml + +```yaml +settings: + concurrency: 5 + user_agent: "Mozilla/5.0 ..." + output_root: "data/raw" + +sources: + - id: "reuters" + name: "Reuters" + enabled: true + homepage: "https://www.reuters.com/business/" + article_url_pattern: "/[^/]+/[^/]+/" + js_render: false + max_articles_per_run: 30 +``` + +### .env 关键配置 + +```bash +# LLM +LLM_PROVIDER=deepseek +LLM_MODEL=deepseek-v4-flash +DEEPSEEK_API_KEY=sk-xxx + +# Embedding +EMBEDDING_PROVIDER=dashscope +DASHSCOPE_API_KEY=sk-xxx + +# 新闻联播 +XWLB_API_BASE=https://api.doorcome.cn + +# Schedule +SCHEDULE_TIMES=06:30,12:00,18:00,22:00 +SYNC_WAIT_SEC=300 + +# Qdrant +QDRANT_COLLECTION=en_finance_news + +# Overseas → Domestic +SYNC_HOST=user@domestic-server +SYNC_PORT=22 +SYNC_SENTINEL=/tmp/en_news_sync_done +``` + +--- + +## 七、目录结构 + +``` +english-news/ +├── crawler/ # M1 新闻抓取(Crawl4AI, 部署海外) +├── extractor/ # M2 英文正文提取(trafilatura) +├── dedup/ # M3 三层去重 +├── translator/ # M4a 全文翻译(LLM) +├── llm/ # M4b 投资事件抽取 +├── embedding/ # M5 向量生成 +├── vectorstore/ # M6 Qdrant 客户端 +├── xwlb/ # M7 新闻联播数据源 +├── scheduler/ # M8 定时任务 +├── mcp_server/ # M9 MCP 服务 +├── app/ # CLI 入口(en-news) +├── configs/ # sources.yaml / system config +├── prompts/ # LLM Prompt 模板 +├── scripts/ # 部署 & 运维脚本 +│ ├── sync_daemon.sh # 海外 rsync 推送脚本 +│ └── wait_for_sync.py # 国内等待同步完成 +├── tests/ # pytest +├── docs/ # 设计文档 +├── data/ # 数据目录(git ignored) +│ ├── raw/ # M1 产物(海外与国内共享) +│ ├── processed/ # M2 产物 +│ ├── deduped/ # M3 产物 +│ ├── events/ # M4 产物 +│ ├── embeddings/ # M5 产物 +│ ├── xwlb/ # 新闻联播数据(经 LLM 筛选与解读) +│ ├── qdrant_storage/ # M6 Qdrant 本地存储 +│ └── reports/ # 日报 HTML +├── logs/ # 运行日志 +├── pyproject.toml +├── CLAUDE.md +└── README.md +``` + +--- + +## 八、技术选型对比 + +| 模块 | A 股项目 | 英文财经项目 | 差异 | +|------|---------|-------------|------| +| 抓取 | Crawl4AI | Crawl4AI | 一致 | +| 正文提取 | GNE | trafilatura | 英文优化 | +| 翻译 | — | LLM (DeepSeek) | **新增** | +| 去重 | 三层 | 三层 | 一致 | +| LLM 事件 | DeepSeek/Qwen | DeepSeek/Qwen | 一致 | +| Embedding | DashScope | DashScope | 一致 | +| 向量库 | Qdrant | Qdrant | 一致 | +| 新闻联播 | — | api.doorcome.cn + LLM 筛选 | **新增** | +| 调度 | APScheduler | APScheduler | 一致 | +| MCP | FastMCP | FastMCP | 一致 | +| CLI | a-share | en-news | 命名区分 | + +--- + +## 九、开发优先级(建议顺序) + +| 序号 | Milestone | 预估工期 | 依赖 | +|------|-----------|---------|------| +| 1 | M0 项目骨架 | 0.5 天 | — | +| 2 | M1 抓取(海外) | 2 天 | M0 | +| 3 | rsync 同步机制 | 0.5 天 | M1 | +| 4 | M2 正文提取 | 1 天 | 同步就绪 | +| 5 | M3 去重 | 1 天 | M2 | +| 6 | M4 翻译+事件抽取 | 1.5 天 | M3 | +| 7 | M5 向量 + M6 Qdrant | 1 天 | M4 | +| 8 | M7 新闻联播数据源 | 1 天 | — (独立模块) | +| 9 | M8 调度 + 日报 | 1.5 天 | M6, M7 | +| 10 | M9 MCP 服务 | 1 天 | M6 | +| **总计** | | **~11 天** | | + +--- + +## 十、关键风险 + +| 风险 | 缓解措施 | +|------|---------| +| 英文源 JS 渲染复杂 | Crawl4AI 已有 Playwright 支持,优先使用轻量级源 | +| LLM 翻译质量不稳定 | System prompt 约束翻译风格,保留原文备查 | +| rsync 网络不稳定 | 添加重试机制 + 哨兵文件完整性检查 | +| 海外 IP 被封 | 轮换 User-Agent,控制请求频率(≥ 2s/req) | +| DeepSeek API 限流 | 并发控制 + Qwen 备选 provider | + +--- + +> 本计划文件:`~/Downloads/cc-projects/english-news-plan.md` +> +> 最后更新:2026-06-20 diff --git a/extractor/__init__.py b/extractor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/extractor/extractor.py b/extractor/extractor.py new file mode 100644 index 0000000..bfb4675 --- /dev/null +++ b/extractor/extractor.py @@ -0,0 +1,258 @@ +"""英文正文提取引擎 + +策略: +1. 优先 trafilatura 从原始 HTML 提取(除导航/广告/页脚) +2. 回退 Crawl4AI Markdown(如果 HTML 不可用) +3. 元数据提取(发布时间、作者) +所有业务参数从 configs/system.yaml 的 extractor 节读取。 +""" + +import logging +from pathlib import Path + +import trafilatura +import yaml +from htmldate import find_date + +from extractor.models import ProcessedArticle + +logger = logging.getLogger(__name__) + + +def _load_extractor_config() -> dict: + """从 system.yaml 读取 extractor 配置节""" + config_path = Path("configs/system.yaml") + if config_path.exists(): + try: + with open(config_path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + return raw.get("extractor", {}) + except Exception: + logger.warning("读取 system.yaml 失败,使用默认值") + return {} + +_cfg = _load_extractor_config() + +# 最小正文字数阈值 +MIN_CONTENT_WORDS = int(_cfg.get("min_content_words", 50)) + + +def extract_article( + source_id: str, + source_name: str, + url: str, + url_hash: str, + html_path: str, + md_path: str, + crawl_publish_time: str = "", +) -> ProcessedArticle: + """提取单篇文章的正文 + + Args: + source_id: 新闻源 ID + source_name: 新闻源名称 + url: 文章 URL + url_hash: URL SHA256 前 16 位 + html_path: 原始 HTML 文件路径 + md_path: Crawl4AI 生成的 Markdown 文件路径 + crawl_publish_time: RSS/Crawler 提取的发布时间(回退用) + + Returns: + ProcessedArticle + """ + article = ProcessedArticle( + source_id=source_id, + source_name=source_name, + url=url, + url_hash=url_hash, + html_path=html_path, + md_path=md_path, + ) + + html_p = Path(html_path) if html_path else None + md_p = Path(md_path) if md_path else None + + # ── 策略 1: trafilatura 从 HTML 提取 ────────────── + if html_p and html_p.exists(): + try: + html_raw = html_p.read_text(encoding="utf-8") + content = trafilatura.extract( + html_raw, + include_comments=False, + include_tables=False, + no_fallback=False, + favor_precision=True, + ) + if content and _count_words(content) >= MIN_CONTENT_WORDS: + article.content = content.strip() + article.extractor = "trafilatura" + article.word_count = _count_words(article.content) + + # 元数据 + article.title = _extract_title(html_raw) or "" + article.publish_time = ( + _extract_publish_time(html_raw, url=url) + or crawl_publish_time + or "" + ) + article.author = _extract_author(html_raw) or "" + + article.status = "success" + return article + except Exception as e: + logger.warning("[%s] trafilatura 提取失败: %s — %s", source_id, url_hash, e) + + # ── 策略 2: Crawl4AI Markdown 回退 ───────────────── + if md_p and md_p.exists(): + try: + md_content = md_p.read_text(encoding="utf-8") + if _count_words(md_content) >= MIN_CONTENT_WORDS: + article.content = _clean_markdown(md_content) + article.extractor = "crawl4ai_md" + article.word_count = _count_words(article.content) + + # 从 Markdown 中提取元数据(RSS 源会写入) + article.title = _extract_from_md( + md_content, r"^#\s+(.+)$", article.title + ) + article.publish_time = ( + _extract_from_md( + md_content, r"\*\*发布时间\*\*:?\s*([^\n]+)", "" + ) + or _extract_publish_time("", url=url) + or crawl_publish_time + or "" + ) + + article.status = "success" + return article + except Exception as e: + logger.warning("[%s] Markdown 回退失败: %s — %s", source_id, url_hash, e) + + # ── 失败 ───────────────────────────────────────── + article.status = "no_content" + article.error = "No extractable content" + return article + + +# ── 辅助函数 ──────────────────────────────────────── + + +def _count_words(text: str) -> int: + """英文词数统计""" + return len(text.split()) if text else 0 + + +def _extract_title(html: str) -> str: + """从 HTML 标签提取标题""" + import re + m = re.search(r"<title[^>]*>(.*?)", html, re.IGNORECASE | re.DOTALL) + if m: + title = re.sub(r"<[^>]+>", "", m.group(1)) + return title.strip() + return "" + + +def _extract_publish_time(html: str, url: str = "") -> str: + """从 HTML 提取发布时间(含回退策略)。 + + 优先级: + 1. htmldate 提取(元数据/标签/正文) + 2. 等常见标签 + 3. URL 中的日期(/YYYY/MM/DD/ 或 /YYYYMMDD/) + """ + # 策略 1: htmldate + try: + result = find_date(html) + if result: + return result + except Exception: + pass + + # 策略 2: 常见 meta 标签 + if html: + import re + meta_patterns = [ + r']+property=["\']article:published_time["\'][^>]+content=["\']([^"\']+)', + r']+content=["\']([^"\']+)["\'][^>]+property=["\']article:published_time', + r']+name=["\']pubdate["\'][^>]+content=["\']([^"\']+)', + r']+name=["\']publish_date["\'][^>]+content=["\']([^"\']+)', + r']+name=["\']date["\'][^>]+content=["\']([^"\']+)', + r']+datetime=["\']([^"\']+)', + ] + for pat in meta_patterns: + m = re.search(pat, html, re.IGNORECASE) + if m: + date_str = m.group(1).strip() + if date_str: + return date_str + + # 策略 3: URL 中的日期 + if url: + from datetime import datetime as dt + import re + url_patterns = [ + r"/(\d{4})/(\d{2})/(\d{2})/", + r"/(\d{4})(\d{2})(\d{2})/", + r"-(\d{4})(\d{2})(\d{2})(?:[/-]|$)", + r"-(\d{4})-(\d{2})-(\d{2})[/-]", + ] + for pat in url_patterns: + m = re.search(pat, url) + if m: + try: + y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3)) + return dt(y, mo, d).isoformat() + except ValueError: + continue + + return "" + + +def _extract_author(html: str) -> str: + """从 HTML meta 标签提取作者""" + import re + for pat in [ + r']+name=["\']author["\'][^>]+content=["\']([^"\']+)', + r']+content=["\']([^"\']+)["\'][^>]+name=["\']author', + ]: + m = re.search(pat, html, re.IGNORECASE) + if m: + return m.group(1).strip() + return "" + + +def _extract_from_md(md: str, pattern: str, default: str) -> str: + """从 Markdown 内容中按正则提取第一个捕获组,失败返回 default。""" + import re + m = re.search(pattern, md) + return m.group(1).strip() if m else default + + +def _clean_markdown(md: str) -> str: + """清理 Crawl4AI MD 中的导航/广告噪音 + + 规则:移除明显非正文行(短链接行、纯导航行等) + """ + lines = md.split("\n") + cleaned: list[str] = [] + skip_patterns = [ + "ADVERTISEMENT", + "Continue Reading Below", + "Sign in", + "Log in", + "Subscribe", + "Advertisement", + ] + + for line in lines: + stripped = line.strip() + # 跳过明显广告/导航行 + if any(p in stripped for p in skip_patterns): + continue + # 跳过纯导航链接行([text](url) 且整行只有链接) + if stripped.startswith("[") and stripped.endswith(")") and stripped.count("[") <= 3: + continue + cleaned.append(line) + + return "\n".join(cleaned) diff --git a/extractor/models.py b/extractor/models.py new file mode 100644 index 0000000..a894147 --- /dev/null +++ b/extractor/models.py @@ -0,0 +1,22 @@ +"""正文提取数据模型""" + +from pydantic import BaseModel + + +class ProcessedArticle(BaseModel): + """经过正文提取清洗后的文章""" + + source_id: str + source_name: str + url: str + url_hash: str + title: str = "" + content: str = "" # 清洗后的正文(英文) + publish_time: str = "" # ISO 8601 + author: str = "" + word_count: int = 0 + md_path: str = "" # 原始 Markdown 路径 + html_path: str = "" # 原始 HTML 路径 + status: str = "success" # success | no_content | failed + extractor: str = "trafilatura" # trafilatura | crawl4ai_md | none + error: str = "" diff --git a/extractor/pipeline.py b/extractor/pipeline.py new file mode 100644 index 0000000..be2c943 --- /dev/null +++ b/extractor/pipeline.py @@ -0,0 +1,175 @@ +"""正文提取管道:批量处理 raw 目录下的文章""" + +import json +import logging +from datetime import datetime +from pathlib import Path + +from crawler.storage import load_index +from crawler.utils import get_news_day +from extractor.extractor import extract_article +from extractor.models import ProcessedArticle + +logger = logging.getLogger(__name__) + + +def get_raw_data_sources(base_dir: str = "data/raw") -> list[str]: + """扫描 data/raw/ 下所有源 ID + + Args: + base_dir: raw 数据根目录 + + Returns: + 源 ID 列表 + """ + raw_path = Path(base_dir) + if not raw_path.exists(): + return [] + return sorted([ + d.name for d in raw_path.iterdir() + if d.is_dir() and not d.name.startswith(".") + ]) + + +def process_source( + source_id: str, + date_str: str | None = None, +) -> list[ProcessedArticle]: + """处理单个源的文章正文提取 + + Args: + source_id: 新闻源 ID + date_str: 日期 YYYYMMDD,默认当前新闻日 + + Returns: + ProcessedArticle 列表 + """ + if date_str is None: + date_str = get_news_day() + + logger.info("━━━ 正文提取 [%s] %s ━━━", source_id, date_str) + + # 读取 raw index + articles = load_index(source_id, date_str) + if not articles: + logger.warning("[%s] %s 无待处理文章", source_id, date_str) + return [] + + # 输出目录 + out_dir = Path(f"data/processed/{source_id}/{date_str}") + out_dir.mkdir(parents=True, exist_ok=True) + + results: list[ProcessedArticle] = [] + success = 0 + no_content = 0 + failed = 0 + + # 增量:跳过已处理的文章 + skipped = 0 + for article in articles[:]: + out_file = out_dir / f"{article.url_hash}.json" + if out_file.exists(): + articles.remove(article) + skipped += 1 + if skipped > 0: + logger.info("[%s] 增量跳过 %d 篇已处理,剩余 %d 篇", + source_id, skipped, len(articles)) + + for article in articles: + # 确保 html_path / md_path 是绝对路径(相对于 data/raw/) + html_path = str(Path("data/raw") / source_id / date_str / f"{article.url_hash}.html") + md_path = str(Path("data/raw") / source_id / date_str / f"{article.url_hash}.md") + + # 如果 index 中已有路径,优先使用 + if article.html_path: + html_path = article.html_path + if article.md_path: + md_path = article.md_path + + processed = extract_article( + source_id=source_id, + source_name=article.source_name, + url=article.url, + url_hash=article.url_hash, + html_path=html_path, + md_path=md_path, + crawl_publish_time=article.publish_time, + ) + + # 写入 JSON 输出 + out_file = out_dir / f"{article.url_hash}.json" + out_file.write_text( + processed.model_dump_json(indent=2, ensure_ascii=False), + encoding="utf-8", + ) + + results.append(processed) + + if processed.status == "success": + success += 1 + logger.debug("[%s] ✅ %s (%d words, %s)", + source_id, processed.title[:40], processed.word_count, processed.extractor) + elif processed.status == "no_content": + no_content += 1 + logger.warning("[%s] ⚠️ 无正文: %s", source_id, processed.url[:80]) + else: + failed += 1 + + logger.info("[%s] 完成: 成功 %d / 无内容 %d / 失败 %d", + source_id, success, no_content, failed) + + # 写入处理索引 + index_path = out_dir / "index.jsonl" + with open(index_path, "a", encoding="utf-8") as f: + for r in results: + if r.status == "success": + f.write(json.dumps({ + "url_hash": r.url_hash, + "title": r.title, + "word_count": r.word_count, + "extractor": r.extractor, + "publish_time": r.publish_time, + }, ensure_ascii=False) + "\n") + + return results + + +def process_all_sources( + source_filter: str | None = None, + date_str: str | None = None, +) -> dict: + """处理所有源的文章正文提取 + + Args: + source_filter: 可选,只处理指定源 + date_str: 日期,默认当前新闻日 + + Returns: + 统计 dict + """ + if date_str is None: + date_str = get_news_day() + + start_time = datetime.now() + + if source_filter: + sources = [source_filter] if source_filter in get_raw_data_sources() else [] + else: + sources = get_raw_data_sources() + + logger.info("══════ 开始正文提取 %d 个源,日期: %s ══════", len(sources), date_str) + + total = 0 + for src in sources: + results = process_source(src, date_str) + total += len([r for r in results if r.status == "success"]) + + elapsed = (datetime.now() - start_time).total_seconds() + logger.info("══════ 提取完成: %d 篇文章,耗时 %.1f 秒 ══════", total, elapsed) + + return { + "sources_processed": len(sources), + "total_articles": total, + "elapsed_sec": elapsed, + "date": date_str, + } diff --git a/llm/__init__.py b/llm/__init__.py new file mode 100644 index 0000000..8c8ad40 --- /dev/null +++ b/llm/__init__.py @@ -0,0 +1,61 @@ +"""LLM 翻译 + 投资事件抽取模块 (M4)。 + +公共 API: + - load_llm_config / make_sync_client / make_async_client + - translate_and_extract / translate_and_extract_async + - PromptTemplate / parse_translation_json + - translate_all_deduped(批量管道) + - EnTranslatedArticle / EventExtraction / LLMTranslationOutput / Sentiment +""" + +from llm.client import ( + LLMConfig, + load_llm_config, + make_async_client, + make_sync_client, +) +from llm.extractor import ( + DEFAULT_MAX_ATTEMPTS, + MAX_CONTENT_CHARS, + PromptTemplate, + parse_translation_json, + translate_and_extract, + translate_and_extract_async, +) +from llm.models import ( + INTERNATIONAL_EVENT_TYPES, + MAX_IMPORTANCE, + MIN_IMPORTANCE, + EnTranslatedArticle, + EventExtraction, + LLMCallError, + LLMTranslationOutput, + Sentiment, +) +from llm.pipeline import translate_all_deduped + +__all__ = [ + # 客户端 + "LLMConfig", + "load_llm_config", + "make_async_client", + "make_sync_client", + # 翻译+抽取 + "DEFAULT_MAX_ATTEMPTS", + "MAX_CONTENT_CHARS", + "PromptTemplate", + "parse_translation_json", + "translate_and_extract", + "translate_and_extract_async", + # 批量管道 + "translate_all_deduped", + # 模型 + "EnTranslatedArticle", + "EventExtraction", + "INTERNATIONAL_EVENT_TYPES", + "LLMCallError", + "LLMTranslationOutput", + "MAX_IMPORTANCE", + "MIN_IMPORTANCE", + "Sentiment", +] diff --git a/llm/client.py b/llm/client.py new file mode 100644 index 0000000..728aa5e --- /dev/null +++ b/llm/client.py @@ -0,0 +1,131 @@ +"""LLM 客户端抽象与工厂。 + +支持 DeepSeek 和 Qwen(百炼),两者均为 OpenAI 兼容接口,共用 openai SDK。 + +配置来源: + - .env → API Key / Base URL(密钥和地址) + - configs/system.yaml → provider / model / timeout / temperature(功能配置) +""" + +import logging +import os +from dataclasses import dataclass +from pathlib import Path + +import yaml +from openai import AsyncOpenAI, OpenAI + +logger = logging.getLogger(__name__) + +# Provider 默认基址 +_DEEPSEEK_DEFAULT_BASE = "https://api.deepseek.com" +_QWEN_DEFAULT_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1" + +# 默认模型 +_DEEPSEEK_DEFAULT_MODEL = "deepseek-chat" +_QWEN_DEFAULT_MODEL = "qwen-plus" + + +def _load_system_config() -> dict: + """加载 configs/system.yaml 中 llm 段配置。""" + config_path = Path("configs/system.yaml") + if config_path.exists(): + try: + with open(config_path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + return raw.get("llm", {}) + except Exception: + logger.warning("加载 llm 配置失败,使用空配置") + return {} + + +@dataclass +class LLMConfig: + """LLM 调用配置(provider / model / api_key / base_url / 参数)。""" + + provider: str # "deepseek" / "qwen" + model: str + api_key: str + base_url: str + timeout_sec: float = 60.0 + temperature: float = 0.1 + max_tokens: int = 8192 + + def __post_init__(self) -> None: + if not self.api_key: + raise ValueError(f"LLM provider={self.provider} 的 API key 为空") + + +def load_llm_config( + provider: str | None = None, + *, + model: str | None = None, +) -> LLMConfig: + """根据配置文件构造 LLMConfig。 + + provider 为 None 时读 system.yaml llm.provider,默认 deepseek。 + model 为 None 时读 system.yaml 中对应 provider 的 model。 + + Raises: + ValueError: API key 未配置 + """ + config = _load_system_config() + p = (provider or config.get("provider", "deepseek")).lower() + + if p == "deepseek": + api_key = os.environ.get("DEEPSEEK_API_KEY", "") + base = os.environ.get("DEEPSEEK_BASE_URL", _DEEPSEEK_DEFAULT_BASE) + m = model or config.get("deepseek_model", _DEEPSEEK_DEFAULT_MODEL) + elif p in ("qwen", "dashscope"): + api_key = os.environ.get("QWEN_API_KEY") or os.environ.get("DASHSCOPE_API_KEY") or "" + base = os.environ.get("QWEN_BASE_URL", _QWEN_DEFAULT_BASE) + m = model or config.get("qwen_model", _QWEN_DEFAULT_MODEL) + p = "qwen" + else: + raise ValueError(f"未知 LLM provider: {p!r},仅支持 deepseek / qwen") + + if not api_key: + raise ValueError( + f"LLM provider={p} 的 API key 未配置,请检查 .env 中的 " + f"{'DEEPSEEK_API_KEY' if p == 'deepseek' else 'QWEN_API_KEY'}" + ) + + timeout = float(config.get("timeout_sec", 60.0)) + temperature = float(config.get("temperature", 0.1)) + max_tokens = int(config.get("max_tokens", 8192)) + + return LLMConfig( + provider=p, + model=m, + api_key=api_key, + base_url=base, + timeout_sec=timeout, + temperature=temperature, + max_tokens=max_tokens, + ) + + +def make_sync_client(config: LLMConfig) -> OpenAI: + """构造同步 OpenAI 客户端(指向 DeepSeek/Qwen 兼容端点)。""" + logger.info( + "初始化同步 LLM 客户端: provider=%s model=%s base_url=%s", + config.provider, config.model, config.base_url, + ) + return OpenAI( + api_key=config.api_key, + base_url=config.base_url, + timeout=config.timeout_sec, + ) + + +def make_async_client(config: LLMConfig) -> AsyncOpenAI: + """构造异步 OpenAI 客户端(用于批处理高并发)。""" + logger.info( + "初始化异步 LLM 客户端: provider=%s model=%s base_url=%s", + config.provider, config.model, config.base_url, + ) + return AsyncOpenAI( + api_key=config.api_key, + base_url=config.base_url, + timeout=config.timeout_sec, + ) diff --git a/llm/extractor.py b/llm/extractor.py new file mode 100644 index 0000000..1397584 --- /dev/null +++ b/llm/extractor.py @@ -0,0 +1,370 @@ +"""LLM 翻译 + 投资事件抽取主流程。 + +输入: ProcessedArticle(M2/M3 输出) +输出: EnTranslatedArticle(含中英文双语内容 + 抽取事件 + 元信息) + +设计: + 1. 加载 prompts/translation_and_extraction.md,分离 system/user 模板 + 2. 单次 LLM 调用完成翻译 + 事件抽取(节省 token) + 3. 解析 JSON → Pydantic LLMTranslationOutput 强校验 + 重试 + 4. 限制正文长度避免触顶 context window +""" + +import asyncio +import json +import logging +import re +import time +from pathlib import Path + +from openai import AsyncOpenAI, OpenAI + +from extractor.models import ProcessedArticle +from llm.client import LLMConfig +from llm.models import ( + EnTranslatedArticle, + LLMCallError, + LLMTranslationOutput, +) + +logger = logging.getLogger(__name__) + +# Prompt 模板路径 +DEFAULT_PROMPT_PATH = Path("prompts/translation_and_extraction.md") + +# 文章正文截断长度(保守取 8K 字符,避免超出上下文窗口) +MAX_CONTENT_CHARS = 8000 + +# 重试设置 +DEFAULT_MAX_ATTEMPTS = 3 +RETRY_BASE_WAIT_SEC = 1.0 +RETRY_MAX_WAIT_SEC = 8.0 + +# Prompt 中 system / user 分隔标记 +_SYSTEM_SECTION_START = "## System Prompt" +_USER_SECTION_START = "## User Input" + + +class PromptTemplate: + """Prompt 模板加载器。 + + 从 prompts/translation_and_extraction.md 读取模板, + 分离 System Prompt 和 User Input 两部分。 + User Input 支持 {title} / {source_name} / {publish_time} / {content} 占位符。 + """ + + def __init__(self, template_path: str | Path = DEFAULT_PROMPT_PATH) -> None: + self._path = Path(template_path) + raw = self._path.read_text(encoding="utf-8") + + # 分离 system 和 user 两部分 + self._system_prompt, self._user_template = self._parse_template(raw) + + @staticmethod + def _parse_template(raw: str) -> tuple[str, str]: + """解析模板文件,返回 (system_prompt, user_template)。""" + # 找到 ## System Prompt 之后的内容直到 ## User Input + sys_match = re.search( + r"## System Prompt\s*\n(.*?)(?=---\s*\n## User Input)", + raw, re.DOTALL + ) + user_match = re.search( + r"## User Input\s*\n(.*)", + raw, re.DOTALL + ) + + system = sys_match.group(1).strip() if sys_match else "" + user = user_match.group(1).strip() if user_match else raw + + return system, user + + def render(self, article: ProcessedArticle) -> tuple[str, str]: + """渲染 Prompt,返回 (system_prompt, user_prompt)。 + + 对正文做截断处理。 + """ + content = article.content + if len(content) > MAX_CONTENT_CHARS: + logger.debug( + "文章 %s 超长截断: %d → %d", + article.url_hash, len(content), MAX_CONTENT_CHARS, + ) + content = content[:MAX_CONTENT_CHARS] + "\n\n[正文过长已截断]" + + user_prompt = ( + self._user_template + .replace("{title}", article.title) + .replace("{source_name}", article.source_name) + .replace("{publish_time}", article.publish_time or "未知") + .replace("{content}", content) + ) + + return self._system_prompt, user_prompt + + +# --------------------------------------------------------------------------- # +# JSON 提取(LLM 偶尔会包 ```json 围栏) +# --------------------------------------------------------------------------- # + + +def _extract_json_object(text: str) -> str: + """从 LLM 输出中提取首个 JSON 对象字符串(去围栏 / 取首个 {...})。""" + s = text.strip() + if s.startswith("```"): + s = s.strip("`") + if s.lower().startswith("json"): + s = s[4:].lstrip("\n").lstrip() + if s.endswith("```"): + s = s[:-3] + start = s.find("{") + if start < 0: + return s + depth = 0 + for i in range(start, len(s)): + if s[i] == "{": + depth += 1 + elif s[i] == "}": + depth -= 1 + if depth == 0: + return s[start : i + 1] + return s[start:] + + +def parse_translation_json(raw: str) -> LLMTranslationOutput: + """把 LLM 输出文本解析为 LLMTranslationOutput(可能抛 LLMCallError)。""" + payload = _extract_json_object(raw) + try: + obj = json.loads(payload) + except json.JSONDecodeError as e: + raise LLMCallError(f"JSON 解析失败: {e}") from e + if not isinstance(obj, dict): + raise LLMCallError(f"JSON 顶层非对象: {type(obj).__name__}") + try: + return LLMTranslationOutput.model_validate(obj) + except Exception as e: + raise LLMCallError(f"翻译输出 schema 校验失败: {e}") from e + + +# --------------------------------------------------------------------------- # +# 词数计算 +# --------------------------------------------------------------------------- # + + +def _count_zh_chars(text: str) -> int: + """统计中文文本字数(汉字 + 数字 + 字母混合)。""" + # 简单统计:去除空白后长度 + return len(text.replace(" ", "").replace("\n", "").replace("\r", "")) + + +# --------------------------------------------------------------------------- # +# 同步 / 异步 LLM 调用 +# --------------------------------------------------------------------------- # + + +def _call_llm_sync( + client: OpenAI, + config: LLMConfig, + system_prompt: str, + user_prompt: str, +) -> tuple[str, dict[str, int | None]]: + """同步单次 LLM 调用,返回 (raw_text, usage)。""" + resp = client.chat.completions.create( + model=config.model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + temperature=config.temperature, + max_tokens=config.max_tokens, + response_format={"type": "json_object"}, + ) + text = resp.choices[0].message.content or "" + usage = { + "prompt_tokens": getattr(resp.usage, "prompt_tokens", None) if resp.usage else None, + "completion_tokens": ( + getattr(resp.usage, "completion_tokens", None) if resp.usage else None + ), + } + return text, usage + + +async def _call_llm_async( + client: AsyncOpenAI, + config: LLMConfig, + system_prompt: str, + user_prompt: str, +) -> tuple[str, dict[str, int | None]]: + """异步单次 LLM 调用。""" + resp = await client.chat.completions.create( + model=config.model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + temperature=config.temperature, + max_tokens=config.max_tokens, + response_format={"type": "json_object"}, + ) + text = resp.choices[0].message.content or "" + usage = { + "prompt_tokens": getattr(resp.usage, "prompt_tokens", None) if resp.usage else None, + "completion_tokens": ( + getattr(resp.usage, "completion_tokens", None) if resp.usage else None + ), + } + return text, usage + + +# --------------------------------------------------------------------------- # +# 主入口:翻译 + 事件抽取 +# --------------------------------------------------------------------------- # + + +def translate_and_extract( + client: OpenAI, + config: LLMConfig, + article: ProcessedArticle, + *, + template: PromptTemplate | None = None, + max_attempts: int = DEFAULT_MAX_ATTEMPTS, +) -> EnTranslatedArticle: + """同步翻译 + 事件抽取(单篇文章,带重试)。 + + Args: + client: OpenAI 同步客户端 + config: LLM 配置 + article: 待处理的英文新闻 + template: Prompt 模板,默认加载 prompts/translation_and_extraction.md + max_attempts: 最大重试次数 + + Returns: + EnTranslatedArticle 含双语内容 + 事件 + + Raises: + LLMCallError: 所有重试均失败 + """ + tpl = template or PromptTemplate() + system_prompt, user_prompt = tpl.render(article) + + last_err: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + raw, usage = _call_llm_sync(client, config, system_prompt, user_prompt) + output = parse_translation_json(raw) + + # 检查 content_zh 非空 + if not output.content_zh.strip(): + raise LLMCallError("LLM 返回的 content_zh 为空") + + return EnTranslatedArticle( + source_id=article.source_id, + source_name=article.source_name, + url=article.url, + url_hash=article.url_hash, + title=article.title, + title_zh=output.title_zh, + content_en=article.content, + content_zh=output.content_zh, + publish_time=article.publish_time, + word_count=article.word_count, + word_count_zh=_count_zh_chars(output.content_zh), + events=output.events, + provider=config.provider, + model=config.model, + attempts=attempt, + prompt_tokens=usage.get("prompt_tokens"), + completion_tokens=usage.get("completion_tokens"), + ) + + except LLMCallError as e: + last_err = e + logger.warning( + "LLM 翻译抽取失败 url=%s 尝试 %d/%d: %s", + article.url, attempt, max_attempts, e.reason, + ) + except Exception as e: + last_err = e + logger.warning( + "LLM 调用异常 url=%s 尝试 %d/%d: %s: %s", + article.url, attempt, max_attempts, type(e).__name__, e, + ) + + if attempt < max_attempts: + wait = min(RETRY_BASE_WAIT_SEC * (2 ** (attempt - 1)), RETRY_MAX_WAIT_SEC) + time.sleep(wait) + + raise LLMCallError( + f"LLM 翻译抽取放弃,共 {max_attempts} 次尝试: {last_err}", + attempts=max_attempts, + ) + + +async def translate_and_extract_async( + client: AsyncOpenAI, + config: LLMConfig, + article: ProcessedArticle, + *, + template: PromptTemplate | None = None, + max_attempts: int = DEFAULT_MAX_ATTEMPTS, + semaphore: asyncio.Semaphore | None = None, +) -> EnTranslatedArticle: + """异步翻译 + 事件抽取(批处理用),与同步版逻辑等价。""" + tpl = template or PromptTemplate() + system_prompt, user_prompt = tpl.render(article) + + async def _run() -> EnTranslatedArticle: + last_err: Exception | None = None + for attempt in range(1, max_attempts + 1): + try: + raw, usage = await _call_llm_async(client, config, system_prompt, user_prompt) + output = parse_translation_json(raw) + + if not output.content_zh.strip(): + raise LLMCallError("LLM 返回的 content_zh 为空") + + return EnTranslatedArticle( + source_id=article.source_id, + source_name=article.source_name, + url=article.url, + url_hash=article.url_hash, + title=article.title, + title_zh=output.title_zh, + content_en=article.content, + content_zh=output.content_zh, + publish_time=article.publish_time, + word_count=article.word_count, + word_count_zh=_count_zh_chars(output.content_zh), + events=output.events, + provider=config.provider, + model=config.model, + attempts=attempt, + prompt_tokens=usage.get("prompt_tokens"), + completion_tokens=usage.get("completion_tokens"), + ) + + except LLMCallError as e: + last_err = e + logger.warning( + "LLM 翻译抽取失败 url=%s 尝试 %d/%d: %s", + article.url, attempt, max_attempts, e.reason, + ) + except Exception as e: + last_err = e + logger.warning( + "LLM 调用异常 url=%s 尝试 %d/%d: %s: %s", + article.url, attempt, max_attempts, type(e).__name__, e, + ) + + if attempt < max_attempts: + wait = min(RETRY_BASE_WAIT_SEC * (2 ** (attempt - 1)), RETRY_MAX_WAIT_SEC) + await asyncio.sleep(wait) + + raise LLMCallError( + f"LLM 翻译抽取放弃,共 {max_attempts} 次尝试: {last_err}", + attempts=max_attempts, + ) + + if semaphore is None: + return await _run() + async with semaphore: + return await _run() diff --git a/llm/models.py b/llm/models.py new file mode 100644 index 0000000..2e61486 --- /dev/null +++ b/llm/models.py @@ -0,0 +1,144 @@ +"""LLM 翻译 + 投资事件抽取数据模型 (M4)。 + +EnTranslatedArticle 是 M4 最终落盘格式,包含中英文双语内容和抽取的事件。 +EventExtraction 是 LLM JSON 输出直接映射,经 Pydantic 强校验。 +""" + +import re +from datetime import datetime +from enum import StrEnum + +from pydantic import BaseModel, Field, field_validator + + +class Sentiment(StrEnum): + """事件情绪倾向。""" + + POSITIVE = "positive" # 利好 + NEUTRAL = "neutral" # 中性 + NEGATIVE = "negative" # 利空 + + +# 国际财经事件类型(LLM Prompt 中展示) +INTERNATIONAL_EVENT_TYPES: tuple[str, ...] = ( + "财报披露", + "并购收购", + "产品发布", + "监管政策", + "宏观经济", + "央行决议", + "行业动态", + "技术突破", + "高管变动", + "诉讼法律", + "市场异动", + "地缘政治", + "大宗商品", + "外汇波动", + "其他", +) + +# 美股代码正则:1-5 个大写字母 +_US_STOCK_RE = re.compile(r"^[A-Z]{1,5}$") + +MIN_IMPORTANCE = 1 +MAX_IMPORTANCE = 5 + + +class EventExtraction(BaseModel): + """LLM 输出的单个事件,直接映射 JSON。""" + + event_type: str = Field(..., description="事件类型,见 INTERNATIONAL_EVENT_TYPES") + stock_codes: list[str] = Field( + default_factory=list, + description="涉及美股代码,如 AAPL、TSLA;无相关股票时为空", + ) + sentiment: Sentiment = Field(..., description="positive/neutral/negative") + importance: int = Field( + ..., ge=MIN_IMPORTANCE, le=MAX_IMPORTANCE, description="1-5 重要程度" + ) + summary_zh: str = Field( + default="", + max_length=200, + description="一句话中文事件摘要", + ) + + @field_validator("stock_codes") + @classmethod + def _validate_stock_codes(cls, v: list[str]) -> list[str]: + """剔除非美股代码格式、去重、统一大写。""" + cleaned: list[str] = [] + seen: set[str] = set() + for code in v: + s = (code or "").strip().upper() + if not s or not _US_STOCK_RE.match(s): + continue + if s not in seen: + seen.add(s) + cleaned.append(s) + return cleaned + + @field_validator("event_type") + @classmethod + def _normalize_event_type(cls, v: str) -> str: + s = (v or "").strip() + return s if s else "其他" + + +class LLMTranslationOutput(BaseModel): + """LLM 单次调用的完整输出 JSON 映射。""" + + title_zh: str = Field(..., description="中文翻译标题") + content_zh: str = Field(..., description="中文翻译正文") + events: list[EventExtraction] = Field( + default_factory=list, description="提取的投资事件列表" + ) + + +class EnTranslatedArticle(BaseModel): + """M4 最终落盘格式:双语文章 + 抽取事件 + 调用元信息。""" + + # ── 来源标识 ── + source_id: str + source_name: str + url: str + url_hash: str + + # ── 双语内容 ── + title: str = "" # 英文原标题 + title_zh: str = "" # 中文翻译标题 + content_en: str = "" # 英文原文 + content_zh: str = "" # 中文翻译 + publish_time: str = "" # ISO 8601 + word_count: int = 0 # 英文词数 + word_count_zh: int = 0 # 中文译文字数 + + # ── 抽取事件 ── + events: list[EventExtraction] = Field(default_factory=list) + + # ── 调用元信息 ── + provider: str = "" # deepseek / qwen + model: str = "" + translated_at: datetime = Field(default_factory=datetime.now) + attempts: int = 1 # 实际调用次数(含重试) + prompt_tokens: int | None = None + completion_tokens: int | None = None + + def short_summary(self) -> str: + codes = set() + for ev in self.events: + codes.update(ev.stock_codes) + codes_str = ",".join(sorted(codes)[:5]) or "-" + return ( + f"[{self.source_id}] {self.title[:30]}... " + f"→ {len(self.events)}events, stocks: {codes_str}" + ) + + +class LLMCallError(Exception): + """LLM 调用失败(网络 / 解析 / 校验)。""" + + def __init__(self, reason: str, *, attempts: int = 0) -> None: + super().__init__(reason) + self.reason = reason + self.attempts = attempts diff --git a/llm/pipeline.py b/llm/pipeline.py new file mode 100644 index 0000000..a200df9 --- /dev/null +++ b/llm/pipeline.py @@ -0,0 +1,238 @@ +"""批量翻译 + 事件抽取管道。 + +输入: data/deduped/{YYYYMMDD}/uniques/{url_hash}.json(M3 去重后唯一条目) +输出: data/events/{YYYYMMDD}/{url_hash}.json +""" + +import json +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from pathlib import Path + +import yaml + +from crawler.utils import get_news_day +from extractor.models import ProcessedArticle +from llm.client import LLMConfig, load_llm_config, make_sync_client +from llm.extractor import PromptTemplate, translate_and_extract +from llm.models import EnTranslatedArticle, LLMCallError + +logger = logging.getLogger(__name__) + +# 默认并发数 +_DEFAULT_CONCURRENCY = 3 + + +def _load_concurrency() -> int: + """从 system.yaml 读取 LLM 并发配置。""" + config_path = Path("configs/system.yaml") + if config_path.exists(): + try: + with open(config_path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + return int(raw.get("llm", {}).get("concurrency", _DEFAULT_CONCURRENCY)) + except Exception: + pass + return _DEFAULT_CONCURRENCY + + +def _load_deduped_articles( + date_str: str, +) -> list[ProcessedArticle]: + """加载指定日期的去重后唯一条目。 + + Args: + date_str: 日期 YYYYMMDD + + Returns: + ProcessedArticle 列表 + """ + base_dir = Path(f"data/deduped/{date_str}/uniques") + if not base_dir.exists(): + return [] + + articles: list[ProcessedArticle] = [] + for json_file in sorted(base_dir.glob("*.json")): + try: + data = json.loads(json_file.read_text(encoding="utf-8")) + articles.append(ProcessedArticle(**data)) + except (json.JSONDecodeError, Exception) as e: + logger.warning("解析去重文章失败 %s: %s", json_file, e) + + return articles + + +def _process_one( + article: ProcessedArticle, + client, + config: LLMConfig, + template: PromptTemplate, +) -> EnTranslatedArticle | None: + """处理单篇文章的翻译 + 事件抽取。返回 None 表示失败。""" + try: + return translate_and_extract( + client=client, + config=config, + article=article, + template=template, + ) + except LLMCallError as e: + logger.error( + "翻译抽取最终失败 url=%s: %s", article.url, e.reason + ) + return None + except Exception as e: + logger.exception("翻译抽取未预期异常 url=%s: %s", article.url, e) + return None + + +def translate_all_deduped( + date_str: str | None = None, + *, + provider: str | None = None, + model: str | None = None, + concurrency: int | None = None, +) -> dict: + """对去重后的所有唯一条目执行翻译 + 事件抽取。 + + Args: + date_str: 日期 YYYYMMDD,默认当前新闻日 + provider: LLM provider,默认从 system.yaml 读取 + model: LLM model,默认从 system.yaml 读取 + concurrency: 并发数,默认从 system.yaml 读取 + + Returns: + 统计 dict + """ + if date_str is None: + date_str = get_news_day() + + concurrency = concurrency or _load_concurrency() + + logger.info("══════ 开始翻译+事件抽取,日期: %s,并发: %d ══════", + date_str, concurrency) + + # 加载去重后的文章 + articles = _load_deduped_articles(date_str) + if not articles: + logger.warning("去重目录无文章: data/deduped/%s/uniques/", date_str) + return {"date": date_str, "total": 0, "success": 0, "failed": 0, "elapsed_sec": 0} + + # 初始化 LLM 客户端 + config = load_llm_config(provider=provider, model=model) + client = make_sync_client(config) + template = PromptTemplate() + + # 输出目录 + out_dir = Path(f"data/events/{date_str}") + out_dir.mkdir(parents=True, exist_ok=True) + + start_time = datetime.now() + success = 0 + failed = 0 + + # 增量:跳过已翻译的文章 + new_articles = [] + skipped = 0 + for a in articles: + if (out_dir / f"{a.url_hash}.json").exists(): + skipped += 1 + else: + new_articles.append(a) + if skipped > 0: + logger.info("增量跳过 %d 篇已翻译,剩余 %d 篇待处理", skipped, len(new_articles)) + articles = new_articles + + # 并发处理 + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = { + executor.submit(_process_one, article, client, config, template): article + for article in articles + } + + for future in as_completed(futures): + article = futures[future] + try: + result = future.result() + except Exception as e: + logger.error("并发任务异常 url=%s: %s", article.url, e) + failed += 1 + continue + + if result is None: + failed += 1 + continue + + # 写入输出文件 + out_file = out_dir / f"{result.url_hash}.json" + out_file.write_text( + result.model_dump_json(indent=2, ensure_ascii=False), + encoding="utf-8", + ) + success += 1 + logger.info( + "[%s] ✅ %s → %s (%d events, %d zh chars)", + result.source_id, + result.title[:40], + result.title_zh[:30], + len(result.events), + result.word_count_zh, + ) + + elapsed = (datetime.now() - start_time).total_seconds() + + # 写入事件索引 + _write_event_index(date_str, success, failed, elapsed, config) + + logger.info( + "══════ 翻译+事件抽取完成: 成功 %d / 失败 %d / 总计 %d,耗时 %.1f 秒 ══════", + success, failed, len(articles), elapsed, + ) + + return { + "date": date_str, + "total": len(articles), + "success": success, + "failed": failed, + "elapsed_sec": elapsed, + "provider": config.provider, + "model": config.model, + } + + +def _write_event_index( + date_str: str, + success: int, + failed: int, + elapsed_sec: float, + config: LLMConfig, +) -> None: + """写入事件索引文件。 + + Args: + date_str: 日期 + success: 成功数 + failed: 失败数 + elapsed_sec: 耗时(秒) + config: LLM 配置 + """ + out_dir = Path(f"data/events/{date_str}") + out_dir.mkdir(parents=True, exist_ok=True) + + index_data = { + "date": date_str, + "success": success, + "failed": failed, + "elapsed_sec": round(elapsed_sec, 1), + "provider": config.provider, + "model": config.model, + "generated_at": datetime.now().isoformat(), + } + + index_path = out_dir / "index.json" + index_path.write_text( + json.dumps(index_data, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + logger.info("事件索引已写入: %s", index_path) diff --git a/logs/.gitkeep b/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/mcp_server/__init__.py b/mcp_server/__init__.py new file mode 100644 index 0000000..e9078d4 --- /dev/null +++ b/mcp_server/__init__.py @@ -0,0 +1,13 @@ +"""MCP 服务模块 (M8)。 + +暴露 5 个 MCP Tool 给 Claude Code / Cherry Studio 调用: + - search_news: 语义检索 + - search_by_stock: 美股代码检索 + - search_by_sentiment: 情绪过滤检索 + - get_today_events: 当日重要事件 + - get_stats: 系统统计 +""" + +from mcp_server.server import mcp + +__all__ = ["mcp"] diff --git a/mcp_server/server.py b/mcp_server/server.py new file mode 100644 index 0000000..d62573c --- /dev/null +++ b/mcp_server/server.py @@ -0,0 +1,310 @@ +"""MCP 服务 (M8) — FastMCP 国际财经 Deep Research 工具。 + +暴露 5 个 MCP Tool 给 Claude Code / Cherry Studio 调用。 +""" + +import json +import logging +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any + +from mcp.server.fastmcp import FastMCP + +from crawler.utils import get_news_day +from embedding.client import ( + EmbeddingConfig, + embed_batch, + load_embedding_config, + make_embedding_client, +) +from vectorstore.client import VectorStore, make_qdrant_client +from vectorstore.models import SearchFilter + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- # +# 单例 Backend +# --------------------------------------------------------------------------- # + + +@dataclass +class _Backend: + emb_config: EmbeddingConfig + emb_client: Any + vector_store: VectorStore + + +_backend: _Backend | None = None + + +def _get_backend() -> _Backend: + """延迟初始化 MCP 后端(embedder + vector_store)。""" + global _backend + if _backend is None: + config = load_embedding_config() + client = make_embedding_client(config) + qdrant = make_qdrant_client() + store = VectorStore(qdrant) + _backend = _Backend(emb_config=config, emb_client=client, vector_store=store) + logger.info("MCP 后端就绪: embed=%s, qdrant=%d 条", + config.model, store.count()) + return _backend + + +# --------------------------------------------------------------------------- # +# 嵌入 + 检索 +# --------------------------------------------------------------------------- # + + +def _search( + query: str, + top_k: int = 10, + search_filter: SearchFilter | None = None, + score_threshold: float = 0.3, +) -> list[dict[str, Any]]: + """嵌入查询 → Qdrant 检索 → 返回 dict 列表。""" + be = _get_backend() + vectors = embed_batch(be.emb_client, be.emb_config, [query]) + if not vectors: + return [] + results = be.vector_store.query( + query_vector=vectors[0], + top_k=top_k, + search_filter=search_filter, + score_threshold=score_threshold, + ) + return [ + { + "title": r.title, + "title_zh": r.title_zh, + "url": r.url, + "source": r.source_id, + "score": round(r.score, 4), + "publish_time": r.publish_time, + "events": r.events, + "content_zh_preview": r.content_zh_preview, + } + for r in results + ] + + +def _load_today_events(date_str: str | None = None) -> list[dict]: + """加载当日高重要度事件。""" + day = date_str or get_news_day() + ev_dir = Path(f"data/events/{day}") + if not ev_dir.is_dir(): + return [] + + events: list[dict] = [] + for fp in sorted(ev_dir.glob("*.json")): + if fp.name == "index.json": + continue + try: + data = json.loads(fp.read_text(encoding="utf-8")) + for ev in data.get("events", []): + if ev.get("importance", 0) >= 4: + events.append({ + **ev, + "title": data.get("title", ""), + "title_zh": data.get("title_zh", ""), + "url": data.get("url", ""), + "source_id": data.get("source_id", ""), + "publish_time": data.get("publish_time", ""), + }) + except (json.JSONDecodeError, OSError): + pass + return sorted(events, key=lambda e: -e.get("importance", 0)) + + +def _fmt_results(hits: list[dict[str, Any]], query: str) -> str: + """检索结果 → Markdown 格式化。""" + if not hits: + return f"未找到与「{query}」相关的结果。" + + lines = [f"## 🔍 检索结果: {query}", "", f"共 {len(hits)} 条:", ""] + for i, h in enumerate(hits, 1): + title = h.get("title_zh") or h.get("title", "无标题") + sentiment_map = {"positive": "🟢利好", "neutral": "⚪中性", "negative": "🔴利空"} + lines.append(f"### {i}. {title}") + lines.append(f"- 来源: {h['source']} | 相似度: {h['score']}") + lines.append(f"- 🔗 {h.get('url', '')}") + + for ev in h.get("events", []): + s = sentiment_map.get(ev.get("sentiment", ""), "") + codes = ",".join(ev.get("stock_codes", [])[:5]) + code_str = f" [{codes}]" if codes else "" + lines.append( + f" - {s} [{ev.get('event_type', '')}] " + f"重要度{ev.get('importance', '')} {ev.get('summary_zh', '')}{code_str}" + ) + lines.append("") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- # +# MCP 服务器 & 工具 +# --------------------------------------------------------------------------- # + +mcp = FastMCP( + name="国际财经DeepResearch", + instructions="国际财经新闻 Deep Research 知识库。语义检索英文财经新闻的翻译与投资事件分析。", +) + + +# ── Tool 1: search_news ── + + +@mcp.tool() +def search_news(query: str, top_k: int = 10) -> str: + """语义检索国际财经新闻知识库。 + + 参数: + query: 自然语言查询(中文或英文,如 "美联储利率决议" "Apple earnings") + top_k: 返回条数(默认 10) + + 返回: Markdown 格式检索结果,含中英文标题、来源、相似度、事件摘要、URL。 + """ + logger.info("search_news query=%r top_k=%d", query, top_k) + hits = _search(query, top_k=top_k) + return _fmt_results(hits, query) + + +# ── Tool 2: search_by_stock ── + + +@mcp.tool() +def search_by_stock(stock_code: str, top_k: int = 10) -> str: + """检索指定美股代码相关的投资事件。 + + 参数: + stock_code: 美股代码(如 AAPL、TSLA、MSFT,1-5 个大写字母) + top_k: 返回条数(默认 10) + + 返回: Markdown 格式检索结果,含与该股票相关的事件、情绪和重要度。 + """ + code = stock_code.strip().upper() + logger.info("search_by_stock code=%r top_k=%d", code, top_k) + + # 带股票过滤的搜索 + hits = _search(code, top_k=top_k, search_filter=SearchFilter(stock_codes=[code])) + if not hits: + # 降级:纯语义搜索 + logger.info("stock 精确命中 0 条,降级为纯语义搜索") + hits = _search(code, top_k=top_k) + return _fmt_results(hits, f"{code} 相关事件") + + +# ── Tool 3: search_by_sentiment ── + + +@mcp.tool() +def search_by_sentiment(query: str, sentiment: str = "all", top_k: int = 20) -> str: + """按情绪倾向检索新闻。 + + 参数: + query: 自然语言查询 + sentiment: positive(利好) / negative(利空) / neutral(中性) / all(全部,默认) + top_k: 返回条数(默认 20) + + 返回: Markdown 检索结果 + 情绪分布统计。 + """ + logger.info("search_by_sentiment query=%r sentiment=%r", query, sentiment) + + filt = None + if sentiment in ("positive", "negative", "neutral"): + filt = SearchFilter(sentiment=sentiment) + + hits = _search(query, top_k=top_k, search_filter=filt) + + # 情绪统计 + pos = sum(1 for h in hits for ev in h.get("events", []) + if ev.get("sentiment") == "positive") + neg = sum(1 for h in hits for ev in h.get("events", []) + if ev.get("sentiment") == "negative") + neu = sum(1 for h in hits for ev in h.get("events", []) + if ev.get("sentiment") == "neutral") + + header = ( + f"## 📊 情绪趋势: {query}\n\n" + f"共 {len(hits)} 条 | 🟢利好 {pos} | 🔴利空 {neg} | ⚪中性 {neu}\n" + ) + return header + "\n" + _fmt_results(hits, query) + + +# ── Tool 4: get_today_events ── + + +@mcp.tool() +def get_today_events(importance_min: int = 4, limit: int = 15) -> str: + """获取当日重要投资事件。 + + 参数: + importance_min: 最低重要度(默认 4) + limit: 最多返回条数(默认 15) + + 返回: Markdown 格式的当日重要事件列表。 + """ + logger.info("get_today_events importance_min=%d limit=%d", importance_min, limit) + + events = _load_today_events() + events = [e for e in events if e.get("importance", 0) >= importance_min] + events = events[:limit] + + if not events: + return "当日暂无符合条件的重要事件。" + + sentiment_map = {"positive": "🟢利好", "neutral": "⚪中性", "negative": "🔴利空"} + lines = [ + f"## 📅 今日重要事件(重要度 ≥ {importance_min})", + "", + f"共 {len(events)} 条:", + "", + ] + for i, ev in enumerate(events, 1): + s = sentiment_map.get(ev.get("sentiment", ""), "") + codes = ",".join(ev.get("stock_codes", [])[:5]) + code_str = f" [{codes}]" if codes else "" + lines.append(f"### {i}. [{ev.get('event_type', '')}]{s} 重要度{ev.get('importance', '')}") + lines.append(f"- **{ev.get('title_zh', ev.get('title', ''))}**") + lines.append(f"- {ev.get('summary_zh', '')}{code_str}") + lines.append(f"- 来源: {ev.get('source_id', '')} | {ev.get('publish_time', '')[:10]}") + lines.append(f"- 🔗 {ev.get('url', '')}") + lines.append("") + return "\n".join(lines) + + +# ── Tool 5: get_stats ── + + +@mcp.tool() +def get_stats() -> str: + """获取系统统计信息(M1-M6 管道数据、Qdrant 库规模等)。 + + 返回: Markdown 格式的系统统计概览。 + """ + logger.info("get_stats") + + be = _get_backend() + count = be.vector_store.count() + info = be.vector_store.info() + + lines = [ + "## 📊 国际财经 Deep Research 系统统计", + "", + f"**Collection**: {info.name}", + f"**向量总数**: {count}", + f"**嵌入模型**: {be.emb_config.model} ({be.emb_config.dimension} 维)", + f"**Qdrant 模式**: {'文件模式' if not info.exists else '运行中'}", + f"**生成时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}", + "", + "### 可用工具", + "", + "- `search_news` — 语义检索新闻", + "- `search_by_stock` — 按美股代码检索", + "- `search_by_sentiment` — 按情绪过滤检索", + "- `get_today_events` — 获取当日重要事件", + "- `get_stats` — 系统统计概览", + ] + return "\n".join(lines) diff --git a/prompts/translation_and_extraction.md b/prompts/translation_and_extraction.md new file mode 100644 index 0000000..5e9e67a --- /dev/null +++ b/prompts/translation_and_extraction.md @@ -0,0 +1,95 @@ +# 国际财经新闻翻译与投资事件抽取 + +## System Prompt + +你是一位专业的国际财经新闻翻译与分析助手。你的任务是对输入的英文财经新闻完成两件事: + +1. **全文英译中** — 将英文新闻翻译为准确、简洁、专业的中文 +2. **投资事件抽取** — 识别文中涉及的投资相关事件,提取关键信息 + +### 翻译规范 + +- 准确性优先:财经术语翻译准确,保持专业一致性 +- 简洁有力:中文表达清晰简练,不做作,不啰嗦 +- 术语统一: + - Federal Reserve → 美联储 + - ECB → 欧洲央行 + - earnings/revenue → 财报/营收 + - M&A/merger → 并购 + - guidance → 业绩指引 + - SEC filing → SEC 文件 + - IPO → 上市/IPO + - ETF → ETF + - bull/bear → 牛市/熊市 +- 数字、百分比、货币金额保持原样($150M → 1.5亿美元 时注意换算) +- 公司名首次出现保留英文原名+中文翻译,如 Apple(苹果) +- 中文译文字数控制在原文 1.0×–1.5× 范围 + +### 事件抽取规范 + +识别文中涉及的投资相关事件,每个事件包含: + +- **event_type**: 事件类型,从以下类型中选择: + 财报披露、并购收购、产品发布、监管政策、宏观经济、央行决议、 + 行业动态、技术突破、高管变动、诉讼法律、市场异动、地缘政治、 + 大宗商品、外汇波动、其他 + +- **stock_codes**: 涉及的美股代码(1-5 个大写字母,如 AAPL、TSLA、MSFT)。 + 仅提取文中明确提到的上市公司代码。无明确代码时为空数组。 + 同一事件涉及多只股票时,将所有股票代码合并到一个事件的 stock_codes 数组中, + 严禁按股票代码拆分成多个事件。 + +- **sentiment**: 情绪判断 + - positive: 利好(业绩超预期、政策放松、产品成功、合作签约等) + - negative: 利空(业绩不及预期、监管处罚、诉讼、产品失败等) + - neutral: 中性(常规披露、日常动态、无明显偏向) + +- **importance**: 重要程度 1-5 + - 5: 极重大(央行利率决议、重大并购、系统性风险) + - 4: 重大(龙头公司财报、重要政策变化、行业趋势转折) + - 3: 中等(一般公司财报、产品发布、行业分析) + - 2: 较轻(日常动态、常规公告) + - 1: 轻微(辅助信息、背景提及) + +- **summary_zh**: 一句话中文事件摘要(不超过 100 字) + +### 输出格式 + +严格按照以下 JSON 格式输出(不要包含 ```json 标记): + +```json +{ + "title_zh": "中文翻译标题", + "content_zh": "中文翻译正文全文", + "events": [ + { + "event_type": "财报披露", + "stock_codes": ["AAPL"], + "sentiment": "positive", + "importance": 4, + "summary_zh": "苹果第三季度营收超预期,iPhone 销售强劲" + } + ] +} +``` + +- 如果文章没有明确的投资事件,events 为空数组 [] +- 每篇文章最多提取 5 个最重要的事件 +- 一条新闻的多个事件 summary_zh 必须互不相同,禁止生成标题/摘要高度重复的事件 +- 同一则新闻若涉及多类事件,每类只提取一次,不要重复拆分 + +--- + +## User Input + +请处理以下英文财经新闻: + +**标题**: {title} + +**来源**: {source_name} + +**发布时间**: {publish_time} + +**正文**: + +{content} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6fa36fe --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +[project] +name = "en-news" +version = "0.1.0" +description = "国际财经新闻抓取与深度研究平台" +readme = "README.md" +requires-python = ">=3.11,<3.13" +license = { text = "MIT" } +authors = [ + { name = "summer" } +] + +dependencies = [ + "crawl4ai>=0.5.0", + "trafilatura>=2.0.0", + "pydantic>=2.0.0", + "httpx>=0.28.0", + "pyyaml>=6.0", + "apscheduler>=3.10.0", + "fastmcp>=0.1.0", + "qdrant-client>=1.12.0", + "typer>=0.15.0", + "rich>=13.0.0", + "psutil>=7.2.2", + "htmldate>=1.10.0", + "openai>=2.43.0", + "python-dotenv>=1.2.2", + "markdown>=3.10.2", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.25.0", + "pytest-cov>=6.0.0", + "ruff>=0.8.0", +] + +[project.scripts] +en-news = "app.cli:app" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = [ + "crawler", + "extractor", + "dedup", + "translator", + "llm", + "embedding", + "vectorstore", + "scheduler", + "mcp_server", + "app", +] + +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W", "UP"] + +[tool.ruff.lint.per-file-ignores] +"scheduler/reporter.py" = ["E501"] # 内嵌 HTML 模板 CSS 行较长 + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" diff --git a/scheduler/__init__.py b/scheduler/__init__.py new file mode 100644 index 0000000..0e96d56 --- /dev/null +++ b/scheduler/__init__.py @@ -0,0 +1,16 @@ +"""定时调度与日报模块 (M7)。 + +公共 API: + - run_pipeline: 全链路编排 M2→M6 + - generate_report: 每日 HTML 日报生成 +""" + +from scheduler.pipeline import PipelineResult, StepResult, run_pipeline +from scheduler.reporter import generate_report + +__all__ = [ + "PipelineResult", + "StepResult", + "generate_report", + "run_pipeline", +] diff --git a/scheduler/jobs.py b/scheduler/jobs.py new file mode 100644 index 0000000..7e4c106 --- /dev/null +++ b/scheduler/jobs.py @@ -0,0 +1,40 @@ +"""定时任务定义 + +海外 & 国内 crontab 参考。 +实际调度由系统 crontab 或 APScheduler 执行。 +""" + +import logging + +logger = logging.getLogger(__name__) + +# ════════════════════════════════════════════════════ +# 国内 crontab(在 /home/pi/intlnews 目录下执行) +# ════════════════════════════════════════════════════ +# +# 全流程:M1 抓取 → M2→M6 管道 → 日报 +# 每天 06:00 / 12:00 / 18:00 / 22:00 各执行一次 +# 前置条件: +# - Xvfb :99 -screen 0 1280x1024x24 -ac +extension RANDR & (开机自启) +# - ss-local + privoxy 已运行(HTTP 代理 127.0.0.1:3128) +# +# 0 6 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh >> logs/full.log 2>&1 +# 0 12 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh >> logs/full.log 2>&1 +# 0 18 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh >> logs/full.log 2>&1 +# 0 22 * * * cd /home/pi/intlnews && bash scripts/domestic_full.sh >> logs/full.log 2>&1 + + +# ════════════════════════════════════════════════════ +# 任务说明 +# ════════════════════════════════════════════════════ + +JOBS_DOMESTIC = { + "domestic_full_0600": "每天 06:00 M1 抓取 → M2→M6 管道 → 日报", + "domestic_full_1200": "每天 12:00 全流程", + "domestic_full_1800": "每天 18:00 全流程", + "domestic_full_2200": "每天 22:00 全流程", +} + +# 06:00 是新闻日切分点(day_cutoff_hour) +# 海外 05:55 打包的是前一日 06:00 到当日 05:59 的新闻 +# 国内 06:30 拉取时已经是新一轮新闻日的开始 diff --git a/scheduler/pipeline.py b/scheduler/pipeline.py new file mode 100644 index 0000000..a57d175 --- /dev/null +++ b/scheduler/pipeline.py @@ -0,0 +1,231 @@ +"""全链路管道编排 (M7)。 + +串联 M2 → M3 → M4 → M5 → M6,每步失败记录日志但不阻断后续。 +""" + +import logging +import time +from dataclasses import dataclass, field +from datetime import datetime + +from dotenv import load_dotenv + +load_dotenv() # 确保 .env 中的 API Key 被加载到 os.environ + +# 抑制第三方库噪音日志 +logging.getLogger("httpx").setLevel(logging.WARNING) +logging.getLogger("httpcore").setLevel(logging.WARNING) +logging.getLogger("openai").setLevel(logging.WARNING) + +logger = logging.getLogger(__name__) + +# 步骤默认超时(秒) +STEP_TIMEOUTS: dict[str, int] = { + "extract": 300, + "dedup": 120, + "translate": 900, + "embed": 300, + "index": 300, + "report": 60, +} + + +@dataclass +class StepResult: + """单步执行结果。""" + + name: str + success: bool + elapsed_sec: float + message: str = "" + started_at: datetime | None = None + + +@dataclass +class PipelineResult: + """全链路执行结果。""" + + steps: list[StepResult] = field(default_factory=list) + started_at: datetime | None = None + finished_at: datetime | None = None + + @property + def all_success(self) -> bool: + return all(s.success for s in self.steps) + + @property + def success_count(self) -> int: + return sum(1 for s in self.steps if s.success) + + +def run_step_extract(date_str: str) -> StepResult: + """M2: 正文提取。""" + started = datetime.now() + try: + from extractor.pipeline import process_all_sources + stats = process_all_sources(date_str=date_str) + elapsed = (datetime.now() - started).total_seconds() + return StepResult( + name="extract", success=True, elapsed_sec=elapsed, + message=f"{stats['total_articles']} 篇", started_at=started, + ) + except Exception as e: + elapsed = (datetime.now() - started).total_seconds() + logger.exception("M2 正文提取失败") + return StepResult(name="extract", success=False, elapsed_sec=elapsed, + message=str(e)[:200], started_at=started) + + +def run_step_dedup(date_str: str) -> StepResult: + """M3: 三层去重。""" + started = datetime.now() + try: + from dedup.pipeline import dedup_all_sources + stats = dedup_all_sources(date_str=date_str) + elapsed = (datetime.now() - started).total_seconds() + return StepResult( + name="dedup", success=True, elapsed_sec=elapsed, + message=f"唯一 {stats['unique']}/重复 {stats['duplicate']}", started_at=started, + ) + except Exception as e: + elapsed = (datetime.now() - started).total_seconds() + logger.exception("M3 去重失败") + return StepResult(name="dedup", success=False, elapsed_sec=elapsed, + message=str(e)[:200], started_at=started) + + +def run_step_translate(date_str: str) -> StepResult: + """M4: 翻译 + 事件抽取。""" + started = datetime.now() + try: + from llm.pipeline import translate_all_deduped + stats = translate_all_deduped(date_str=date_str) + elapsed = (datetime.now() - started).total_seconds() + return StepResult( + name="translate", success=True, elapsed_sec=elapsed, + message=f"{stats['success']}/{stats['total']} 篇 ({stats.get('provider','')})", + started_at=started, + ) + except Exception as e: + elapsed = (datetime.now() - started).total_seconds() + logger.exception("M4 翻译失败") + return StepResult(name="translate", success=False, elapsed_sec=elapsed, + message=str(e)[:200], started_at=started) + + +def run_step_embed(date_str: str) -> StepResult: + """M5: 向量生成。""" + started = datetime.now() + try: + from embedding.pipeline import embed_all_events + stats = embed_all_events(date_str=date_str) + elapsed = (datetime.now() - started).total_seconds() + return StepResult( + name="embed", success=True, elapsed_sec=elapsed, + message=f"{stats['success']}/{stats['total']} 篇", started_at=started, + ) + except Exception as e: + elapsed = (datetime.now() - started).total_seconds() + logger.exception("M5 向量生成失败") + return StepResult(name="embed", success=False, elapsed_sec=elapsed, + message=str(e)[:200], started_at=started) + + +def run_step_index(date_str: str) -> StepResult: + """M6: Qdrant 入库。""" + started = datetime.now() + try: + from vectorstore.pipeline import ingest_all_embeddings + stats = ingest_all_embeddings(date_str=date_str) + elapsed = (datetime.now() - started).total_seconds() + return StepResult( + name="index", success=True, elapsed_sec=elapsed, + message=f"{stats['ingested']}/{stats['total']} 条", started_at=started, + ) + except Exception as e: + elapsed = (datetime.now() - started).total_seconds() + logger.exception("M6 入库失败") + return StepResult(name="index", success=False, elapsed_sec=elapsed, + message=str(e)[:200], started_at=started) + + +def run_step_report(date_str: str) -> StepResult: + """日报生成。""" + started = datetime.now() + try: + from scheduler.reporter import generate_report + path = generate_report() + elapsed = (datetime.now() - started).total_seconds() + ok = path is not None + return StepResult( + name="report", success=ok, elapsed_sec=elapsed, + message=str(path) if path else "无数据", started_at=started, + ) + except Exception as e: + elapsed = (datetime.now() - started).total_seconds() + logger.exception("日报生成异常") + return StepResult(name="report", success=False, elapsed_sec=elapsed, + message=str(e)[:200], started_at=started) + + +def run_pipeline( + date_str: str, + *, + steps: list[str] | None = None, + skip_report: bool = False, +) -> PipelineResult: + """串联执行全链路 M2→M6(+ 可选日报)。 + + Args: + date_str: YYYYMMDD 日期 + steps: 可选步骤列表,默认全部 + skip_report: 是否跳过日报生成 + + Returns: + PipelineResult + """ + if steps is None: + steps = ["extract", "dedup", "translate", "embed", "index"] + if not skip_report: + steps.append("report") + + step_funcs = { + "extract": run_step_extract, + "dedup": run_step_dedup, + "translate": run_step_translate, + "embed": run_step_embed, + "index": run_step_index, + "report": run_step_report, + } + + result = PipelineResult(started_at=datetime.now()) + + for name in steps: + func = step_funcs.get(name) + if func is None: + logger.warning("未知步骤: %s,跳过", name) + result.steps.append(StepResult(name=name, success=False, elapsed_sec=0, + message=f"未知步骤: {name}")) + continue + + logger.info("── 步骤 %s 开始 ──", name) + sr = func(date_str) + result.steps.append(sr) + + flag = "✅" if sr.success else "❌" + logger.info("── 步骤 %s %s (%.1fs) %s", name, flag, sr.elapsed_sec, sr.message) + + if not sr.success: + logger.warning("步骤 %s 失败,后续步骤继续", name) + + time.sleep(0.5) + + result.finished_at = datetime.now() + total = (result.finished_at - result.started_at).total_seconds() if result.started_at else 0 + + logger.info( + "Pipeline 完成: %d/%d 步骤成功,总耗时 %.0fs", + result.success_count, len(result.steps), total, + ) + + return result diff --git a/scheduler/reporter.py b/scheduler/reporter.py new file mode 100644 index 0000000..2fa2311 --- /dev/null +++ b/scheduler/reporter.py @@ -0,0 +1,952 @@ +"""每日 AI 摘要日报生成器。 + +输出 HTML 日报,包含: + 一、AI 摘要(LLM 根据当日重要事件生成) + 二、重要事件(importance ≥ 4,最多 20 篇) + 三、数据总览(管道统计、情绪分布、重要度分布、事件类型分布) +""" + +import json +import logging +from collections import Counter +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import markdown +import yaml + +logger = logging.getLogger(__name__) + +_MAX_HIGH_EVENTS = 30 +_REPORT_DIR = Path("data/reports") + + +def _load_source_names() -> dict[str, str]: + """从 sources.yaml 加载源名称映射。""" + try: + with open("configs/sources.yaml", encoding="utf-8") as f: + data = yaml.safe_load(f) + return {s["id"]: s["name"] for s in (data.get("sources") or []) if s.get("id")} + except Exception: + return {} + + +def _source_name(src_id: str) -> str: + return _load_source_names().get(src_id, src_id) + + +def _load_domain_name_map() -> dict[str, str]: + """构建域名 → 来源展示名称映射。 + + 来源: + 1. sources.yaml 中各源 homepage 的域名 + 2. 已知域名别名(如 investinglive.com → ForexLive) + """ + domain_map: dict[str, str] = {} + try: + with open("configs/sources.yaml", encoding="utf-8") as f: + data = yaml.safe_load(f) + for s in data.get("sources") or []: + name = s.get("name", "") + homepage = s.get("homepage", "") + if not name or not homepage: + continue + try: + from urllib.parse import urlparse + domain = urlparse(homepage).netloc.removeprefix("www.") + if domain: + domain_map[domain] = name + except Exception: + pass + except Exception: + pass + + return domain_map + + +def _url_source_label(url: str, source_id: str = "") -> str: + """从 URL 提取域名,映射为来源展示名称。 + + 确保日报中链接域名与来源标注一致。 + 未知域名直接显示域名(去 www 前缀)。 + + Args: + url: 文章 URL + source_id: 回退用的 source_id + + Returns: + 来源展示名称 + """ + if not url: + return _source_name(source_id) if source_id else "?" + + try: + from urllib.parse import urlparse + domain = urlparse(url).netloc.removeprefix("www.") + except Exception: + return _source_name(source_id) if source_id else "?" + + if not domain: + return _source_name(source_id) if source_id else "?" + + domain_map = _load_domain_name_map() + return domain_map.get(domain, domain) + + +# 日报覆盖时间窗口(小时) +_REPORT_WINDOW_HOURS = 25 + + +def _dates_in_window(now: datetime) -> list[str]: + """返回 now - 25h 到 now 之间覆盖的所有 YYYYMMDD 日期。 + + 跨天场景:now=06-21 03:00 → now-25h=06-20 02:00 → 返回 ["20260620", "20260621"] + """ + start = now - timedelta(hours=_REPORT_WINDOW_HOURS) + dates: list[str] = [] + current = start.replace(hour=0, minute=0, second=0, microsecond=0) + end = now.replace(hour=23, minute=59, second=59) + while current <= end: + dates.append(current.strftime("%Y%m%d")) + current += timedelta(days=1) + return dates + + +def _try_parse_time(time_str: str) -> datetime | None: + """尝试解析多种 ISO 8601 变体,失败返回 None。""" + if not time_str or not time_str.strip(): + return None + s = time_str.strip() + # 按长度尝试常见格式 + candidates = [s] + if "T" not in s and len(s) == 8: # YYYYMMDD + candidates.append(f"{s[:4]}-{s[4:6]}-{s[6:8]}T00:00:00") + elif "T" not in s and len(s) == 10: # YYYY-MM-DD + candidates.append(f"{s}T00:00:00") + for c in candidates: + try: + dt = datetime.fromisoformat(c) + # 统一转为 UTC-aware: + # - 有时区 → astimezone 转为 UTC + # - 无时区 → 假设为 UTC(多数财经新闻 API 使用 UTC) + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + except ValueError: + continue + return None + + +def _extract_date_from_url(url: str) -> str: + """从 URL 中提取日期(回退策略)。 + + 匹配模式: /YYYY/MM/DD/、/YYYYMMDD/、-YYYYMMDD、-YYYY-MM-DD + """ + import re + patterns = [ + r"/(\d{4})/(\d{2})/(\d{2})/", # /2026/06/19/ + r"/(\d{4})(\d{2})(\d{2})/", # /20260619/ + r"-(\d{4})(\d{2})(\d{2})(?:[/-]|$)", # -20260619/ or -20260619- or -20260619 + r"-(\d{4})-(\d{2})-(\d{2})[/-]", # -2026-06-19/ + ] + for pat in patterns: + m = re.search(pat, url) + if m: + y, mo, d = m.group(1), m.group(2), m.group(3) + try: + dt = datetime(int(y), int(mo), int(d)) + return dt.isoformat() + except ValueError: + continue + return "" + + +def _load_events_window(now: datetime) -> list[dict]: + """加载过去 25 小时内所有事件文章(跨天聚合)。 + + Returns: + 文章 dict 列表,含 title/title_zh/url/source_id/events 等 + """ + # cutoff 使用 UTC-aware,与 _try_parse_time 返回的 UTC datetime 对齐 + cutoff = now.astimezone(timezone.utc) - timedelta(hours=_REPORT_WINDOW_HOURS) + articles: list[dict] = [] + skipped_empty_pt = 0 + for day_str in _dates_in_window(now): + ev_dir = Path(f"data/events/{day_str}") + if not ev_dir.is_dir(): + continue + for fp in sorted(ev_dir.glob("*.json")): + if fp.name == "index.json": + continue + try: + data = json.loads(fp.read_text(encoding="utf-8")) + # 时间过滤:publish_time 在 25 小时内 + pt_str = data.get("publish_time", "") + pt = _try_parse_time(pt_str) + + # 回退:尝试从 URL 提取日期 + if pt is None: + url_pt = _extract_date_from_url(data.get("url", "")) + pt = _try_parse_time(url_pt) + if pt is not None: + logger.debug( + "publish_time 缺失,从 URL 提取: %s → %s", + data.get("url", "")[:60], url_pt, + ) + + # 最后回退:使用事件目录日期(粗略近似) + if pt is None: + dir_pt = _try_parse_time(day_str) + if dir_pt is not None: + logger.debug( + "publish_time 缺失,使用目录日期回退: %s → %s", + fp.name, day_str, + ) + pt = dir_pt + + # 仍然无法确定时间 → 排除 + if pt is None: + skipped_empty_pt += 1 + continue + + if pt < cutoff: + continue + articles.append(data) + except (json.JSONDecodeError, OSError): + pass + if skipped_empty_pt > 0: + logger.warning( + "日报时间过滤: 排除 %d 篇 publish_time 缺失的文章(窗口 %dh)", + skipped_empty_pt, _REPORT_WINDOW_HOURS, + ) + return articles + + +def _collect_stats_window(now: datetime) -> dict: + """收集过去 25 小时全链路统计数据(跨天聚合)。""" + proc_count = 0 + deduped = 0 + emb_count = 0 + raw_by_source: dict[str, int] = {} + + for day_str in _dates_in_window(now): + for d in Path("data/processed").glob(f"*/{day_str}"): + proc_count += len(list(d.glob("*.json"))) + dedup_dir = Path(f"data/deduped/{day_str}/uniques") + if dedup_dir.is_dir(): + deduped += len(list(dedup_dir.glob("*.json"))) + emb_dir = Path(f"data/embeddings/{day_str}") + if emb_dir.is_dir(): + emb_count += len(list(emb_dir.glob("*.json"))) + for idx in Path("data/raw").glob(f"*/{day_str}/index.jsonl"): + src = idx.parent.parent.name + n = sum(1 for _ in open(idx, encoding="utf-8")) + name = _source_name(src) + raw_by_source[name] = raw_by_source.get(name, 0) + n + + qdrant_count = 0 + try: + from vectorstore.client import VectorStore, make_qdrant_client + c = make_qdrant_client() + s = VectorStore(c) + qdrant_count = s.count() + s.close() + except Exception: + pass + + return { + "proc": proc_count, + "deduped": deduped, + "emb_count": emb_count, + "qdrant_count": qdrant_count, + "raw_total": sum(raw_by_source.values()), + "raw_by_source": raw_by_source, + } + + +# AI 摘要分批阈值:单批最多处理的事件条数 +_MAX_EVENTS_PER_BATCH = 10 +# 单批摘要目标字数(软上限,最后一条不允许截断) +_BATCH_SUMMARY_TARGET_CHARS = 300 +# 最终摘要目标字数(软上限,最后一条不允许截断) +_FINAL_SUMMARY_TARGET_CHARS = 500 + + +# LLM 客户端缓存(避免每次调用都创建新客户端) +_llm_client_cache: dict = {} + + +def _call_llm_simple( + system_prompt: str, + user_prompt: str, + max_tokens: int = 600, + max_retries: int = 2, +) -> str: + """封装 LLM 调用,带重试和客户端复用。 + + Args: + system_prompt: system role 内容 + user_prompt: user role 内容 + max_tokens: 最大输出 token + max_retries: 最大重试次数(不含首次调用) + + Returns: + LLM 输出文本;所有重试均失败返回空字符串 + """ + import time as _time + + # 复用客户端(同 provider/model 只创建一次) + cache_key = "default" + if cache_key not in _llm_client_cache: + from llm.client import load_llm_config, make_sync_client + _llm_client_cache["config"] = load_llm_config() + _llm_client_cache[cache_key] = make_sync_client(_llm_client_cache["config"]) + + config = _llm_client_cache["config"] + client = _llm_client_cache[cache_key] + + last_err: str = "" + for attempt in range(1, max_retries + 2): # 首次 + max_retries 次重试 + try: + resp = client.chat.completions.create( + model=config.model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + temperature=0.3, + max_tokens=max_tokens, + ) + content = (resp.choices[0].message.content or "").strip() + if content: + return content + # 内容为空也视为失败 + last_err = "LLM 返回空内容" + logger.warning( + "LLM 返回空内容 (attempt %d/%d, max_tokens=%d)", + attempt, max_retries + 1, max_tokens, + ) + except Exception as e: + last_err = f"{type(e).__name__}: {e}" + logger.warning( + "LLM 调用失败 (attempt %d/%d): %s", + attempt, max_retries + 1, last_err, + ) + + if attempt <= max_retries: + wait = min(1.0 * (2 ** (attempt - 1)), 4.0) + _time.sleep(wait) + + logger.error("LLM 调用最终失败(共 %d 次尝试): %s", max_retries + 1, last_err) + return "" + + +def _is_truncated(text: str) -> bool: + """检测文本是否被截断(最后一条要点不完整)。 + + 判断标准: + - 以完整中文/英文句末标点结尾 → 未截断 + - 以换行结尾 → 未截断(LLM 自然换行说明已写完当前要点) + - 最后一行是 "- " 开头的要点但无句末标点 → 截断 + - 其他不以标点结尾的情况 → 截断 + """ + if not text or not text.strip(): + return False + _SENTENCE_END = ("。", "!", "?", ")", ")", "」", "』", "”", ".", "!", "?") + # 以完整句末标点结尾 → 未截断 + if text.rstrip().endswith(_SENTENCE_END): + return False + # 以换行结尾 → 大概率未截断(先检查原始文本,避免 rstrip 去掉换行) + if text.endswith("\n"): + return False + # 最后一行以 "- " 开头但无句末标点 → 截断 + last_line = text.rstrip().split("\n")[-1].strip() + if last_line.startswith("- ") and not last_line.endswith(_SENTENCE_END): + return True + # 不以任何已知完整标记结尾 → 截断 + return True + + +def _build_event_lines(articles: list[dict]) -> list[str]: + """从文章列表构建事件摘要行列表(仅 importance ≥ 4)。""" + lines: list[str] = [] + for a in articles: + for ev in a.get("events", []): + if ev.get("importance", 0) >= 4: + sentiment_label = { + "positive": "利好", "negative": "利空", "neutral": "中性", + }.get(ev.get("sentiment", ""), "") + codes = ",".join(ev.get("stock_codes", [])[:3]) + code_str = f" [{codes}]" if codes else "" + lines.append( + f"- [{sentiment_label}][{ev.get('event_type', '')}] " + f"{a.get('title_zh', a.get('title', ''))}。{ev.get('summary_zh', '')}{code_str}" + ) + return lines + + +def _fallback_summary(events_info: list[str], max_items: int = 8) -> str: + """LLM 全部失败时的规则回退摘要:直接列出当日 Top 事件。 + + 不依赖 LLM,直接从 events_info 提取前 max_items 条展示。 + """ + if not events_info: + return "⚠️ 暂无重要事件数据" + + top = events_info[:max_items] + lines = [ + f"- {line.lstrip('- ')}" + for line in top + ] + header = ( + "⚠️ AI 摘要暂时无法生成(LLM 服务异常),以下是当日重要事件原始列表:\n\n" + ) + return header + "\n".join(lines) + + +def _generate_ai_summary(articles: list[dict], day_str: str) -> str: + """LLM 生成日报 AI 摘要(≤ 500 字要点列表)。 + + 当高重要度事件超过 _MAX_EVENTS_PER_BATCH 条时,分批生成部分摘要, + 最后合并为最终摘要。避免单次输入内容过长导致 LLM 失败。 + """ + events_info = _build_event_lines(articles) + + if not events_info: + return "" + + # ── 事件少:直接生成 ── + if len(events_info) <= _MAX_EVENTS_PER_BATCH: + input_text = "\n".join(events_info) + prompt = f"""以下是国际财经新闻的当日重要事件摘要 ({day_str}): + +{input_text} + +请用要点总结,每条以 "- " 开头,要求: +1. 前 3 条为当日影响最大的事件,说明为什么重要 +2. 市场情绪基调(利好/利空/中性分布) +3. 值得持续关注的行业、主题或地缘政治动向 +4. 纯要点,不要开场白/结束语/标题 +5. 约 {_FINAL_SUMMARY_TARGET_CHARS} 字左右,但最后一条要点必须完整输出,严禁截断 + +直接输出要点列表:""" + + result = _call_llm_simple( + "你是国际财经日报撰写助手,输出简洁、有洞察的新闻摘要。", + prompt, + max_tokens=800, + ) + if result: + # 截断检测:若被截断,以更高 max_tokens 重试一次 + if _is_truncated(result): + logger.warning("AI 摘要疑似截断,以更高 max_tokens 重试") + retry_prompt = prompt + "\n\n⚠️ 注意:上次输出被截断了,请确保最后一条要点完整结束。" + retry_result = _call_llm_simple( + "你是国际财经日报撰写助手。务必确保输出完整,不以不完整句子结尾。", + retry_prompt, + max_tokens=1200, + ) + if retry_result: + return retry_result + return result + logger.warning("AI 摘要:单次 LLM 调用失败,使用规则回退") + return _fallback_summary(events_info) + + # ── 事件多:分批处理 ── + logger.info( + "AI 摘要分批处理: 共 %d 条高重要度事件,每批 ≤ %d 条", + len(events_info), _MAX_EVENTS_PER_BATCH, + ) + + # 分批生成部分摘要 + partial_summaries: list[str] = [] + for batch_idx in range(0, len(events_info), _MAX_EVENTS_PER_BATCH): + batch = events_info[batch_idx:batch_idx + _MAX_EVENTS_PER_BATCH] + batch_num = batch_idx // _MAX_EVENTS_PER_BATCH + 1 + total_batches = (len(events_info) + _MAX_EVENTS_PER_BATCH - 1) // _MAX_EVENTS_PER_BATCH + + batch_text = "\n".join(batch) + prompt = f"""以下是国际财经新闻的当日重要事件摘要 第 {batch_num}/{total_batches} 批 ({day_str}): + +{batch_text} + +请用要点总结本批事件,每条以 "- " 开头,要求: +1. 提取本批最重要的 3-5 条事件 +2. 说明这些事件的市场影响方向 +3. 纯要点,不要开场白/结束语/标题 +4. 约 {_BATCH_SUMMARY_TARGET_CHARS} 字左右,但最后一条要点必须完整输出,严禁截断 + +直接输出要点列表:""" + + result = _call_llm_simple( + "你是国际财经日报撰写助手,输出简洁、有洞察的新闻摘要。", + prompt, + max_tokens=500, + ) + if result: + # 截断检测:若被截断,以更高 max_tokens 重试一次 + if _is_truncated(result): + logger.warning( + "AI 摘要分批: 第 %d/%d 批疑似截断,重试", batch_num, total_batches, + ) + retry_prompt = prompt + "\n\n⚠️ 注意:上次输出被截断了,请确保最后一条要点完整结束。" + retry_result = _call_llm_simple( + "你是国际财经日报撰写助手。务必确保输出完整,不以不完整句子结尾。", + retry_prompt, + max_tokens=800, + ) + if retry_result: + partial_summaries.append(retry_result) + logger.info("AI 摘要分批: 第 %d/%d 批重试完成 (%d 字)", + batch_num, total_batches, len(retry_result)) + continue + partial_summaries.append(result) + logger.info("AI 摘要分批: 第 %d/%d 批完成 (%d 字)", + batch_num, total_batches, len(result)) + else: + logger.warning("AI 摘要分批: 第 %d/%d 批失败", batch_num, total_batches) + + if not partial_summaries: + # 全部批次 LLM 调用失败 → 回退:直接列出 Top 事件 + logger.warning("AI 摘要:所有分批 LLM 调用均失败,使用规则回退") + return _fallback_summary(events_info) + + # ── 合并部分摘要为最终摘要 ── + merged_input = "\n\n---\n\n".join( + f"第 {i+1} 批摘要:\n{s}" for i, s in enumerate(partial_summaries) + ) + merge_prompt = f"""以下是当日国际财经新闻的多批摘要 ({day_str}),请合并为一份简洁的最终日报摘要: + +{merged_input} + +请合并为要点总结,每条以 "- " 开头,要求: +1. 前 3 条为当日影响最大的事件,说明为什么重要 +2. 市场情绪基调(利好/利空/中性分布) +3. 值得持续关注的行业、主题或地缘政治动向 +4. 纯要点,不要开场白/结束语/标题 +5. 约 {_FINAL_SUMMARY_TARGET_CHARS} 字左右,但最后一条要点必须完整输出,严禁截断 + +直接输出要点列表:""" + + result = _call_llm_simple( + "你是国际财经日报撰写助手,输出简洁、有洞察的新闻摘要。合并多批摘要时注意去重。", + merge_prompt, + max_tokens=1000, + ) + if result: + # 截断检测:若被截断,以更高 max_tokens 重试一次 + if _is_truncated(result): + logger.warning("AI 摘要合并疑似截断,以更高 max_tokens 重试") + retry_merge_prompt = merge_prompt + "\n\n⚠️ 注意:上次输出被截断了,请确保最后一条要点完整结束。" + retry_result = _call_llm_simple( + "你是国际财经日报撰写助手。务必确保输出完整,不以不完整句子结尾。合并多批摘要时注意去重。", + retry_merge_prompt, + max_tokens=1500, + ) + if retry_result: + return retry_result + return result + # 合并失败 → 拼接所有部分摘要作为回退 + logger.warning("AI 摘要:合并 LLM 调用失败,使用部分摘要拼接") + return "⚠️ AI 摘要合并失败,以下为各批次原始摘要:\n\n" + "\n\n".join(partial_summaries) + + +def generate_report() -> Path | None: + """生成 HTML 日报(覆盖过去 25 小时数据)。 + + 命名规则: intl_news_daily_{YYYYMMDD_HHMMSS}.html — 支持一天多份日报。 + + Returns: + HTML 文件路径,无数据时返回 None + """ + now = datetime.now() + ts = now.strftime("%Y%m%d_%H%M%S") + date_str = now.strftime("%Y%m%d") # 用于上传目录 + logger.info("生成日报: %s(窗口: 过去 %d 小时)", ts, _REPORT_WINDOW_HOURS) + + # 加载过去 25 小时数据 + articles = _load_events_window(now) + stats = _collect_stats_window(now) + + if not articles and stats["raw_total"] == 0: + logger.warning("过去 %d 小时无数据,跳过日报生成", _REPORT_WINDOW_HOURS) + return None + + # 收集事件统计 + all_events: list[dict] = [] + sentiments: Counter = Counter() + importances: Counter = Counter() + event_types: Counter = Counter() + sources: Counter = Counter() + + for a in articles: + sources[a.get("source_id", "?")] += 1 + for ev in a.get("events", []): + all_events.append({**ev, "article": a}) + sentiments[ev.get("sentiment", "?")] += 1 + importances[ev.get("importance", 0)] += 1 + event_types[ev.get("event_type", "?")] += 1 + + # 高重要度事件(importance ≥ 4,不足逐级回退) + def _get_high(evs, threshold): + return sorted( + [e for e in evs if e.get("importance", 0) >= threshold], + key=lambda e: -e.get("importance", 0), + ) + + high = _get_high(all_events, 4) + hi_threshold = 4 + if len(high) < 3: + high = _get_high(all_events, 3) + hi_threshold = 3 + if len(high) < 3: + high = sorted(all_events, key=lambda e: -e.get("importance", 0)) + hi_threshold = 0 + # 事件级去重:同一 URL + 同一标题 → 合并 + high = _dedup_events(high) + high = high[:_MAX_HIGH_EVENTS] + + # AI 摘要 + ai_summary = _generate_ai_summary(articles, ts) + + # 渲染 HTML + html = _render_html(ts, stats, articles, high, hi_threshold, + sentiments, importances, event_types, sources, ai_summary) + + # 本地保存(文件名含时间戳) + _REPORT_DIR.mkdir(parents=True, exist_ok=True) + html_path = _REPORT_DIR / f"intl_news_daily_{ts}.html" + html_path.write_text(html, encoding="utf-8") + logger.info("日报已保存: %s (%d KB)", html_path, len(html) // 1024) + + # 自动上传到日期子目录 + _upload_report(html_path, date_str) + + return html_path + + +# --------------------------------------------------------------------------- # +# 上传 +# --------------------------------------------------------------------------- # + + +def _load_report_config() -> dict: + """从 system.yaml 加载 report 段配置。""" + config_path = Path("configs/system.yaml") + if config_path.exists(): + try: + with open(config_path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + return raw.get("report", {}) + except Exception: + pass + return {} + + +def _upload_report(html_path: Path, day_str: str) -> bool: + """上传日报到 Web 服务器(配置来自 system.yaml)。""" + import subprocess + + config = _load_report_config() + host = config.get("upload_host", "").strip() + base = config.get("upload_path", "").strip() + + if not host or not base: + logger.debug("未配置 report.upload_host/upload_path,跳过上传") + return False + + remote_dir = f"{host}:{base}/{day_str}/" + + try: + # 创建远程目录 + subprocess.run( + ["ssh", host, f"mkdir -p {base}/{day_str}/"], + timeout=15, capture_output=True, text=True, + ) + # 上传 + subprocess.run( + ["scp", str(html_path), remote_dir], + timeout=30, capture_output=True, text=True, + ) + logger.info( + "日报已上传: https://echart.doorcome.cn/research/%s/", day_str + ) + return True + except Exception as e: + logger.warning("日报上传失败(不阻塞): %s", e) + return False + + +# --------------------------------------------------------------------------- # +# HTML 渲染 +# --------------------------------------------------------------------------- # + +_HTML_TEMPLATE = """ + + + + +国际财经 Deep Research 日报 — {date} + + + +
+
+

🌍 国际财经 Deep Research 日报

+

{date} · 生成于 {generated_at}

+
+
+
+ + +

一、🤖 AI 摘要

+
{ai_summary}
+ + +

二、🔥 重要事件 (importance ≥ {hi_threshold}, {high_count} 条)

+{events_table} + + +

三、📊 数据总览

+ +

3.1 M1→M6 管道

+
+
{raw_total}
M1 原始文章
+
{proc}
M2 正文提取
+
{deduped}
M3 去重唯一
+
{emb_count}
M5 向量
+
{qdrant_count}
M6 Qdrant
+
+ +

3.2 情绪分布 (当日事件)

+{sentiment_section} + +

3.3 重要度分布

+{importance_table} + +

3.4 事件类型 TOP 10

+{event_type_table} + +

3.5 文章来源分布

+{source_table} + +
+
+
+

国际财经 Deep Research 私有投研平台 · 自动生成于 {generated_at}

+
+
+ +""" + + +def _md_to_html(text: str) -> str: + """Markdown → HTML(使用 python-markdown,开启常用扩展)。""" + if not text.strip(): + return "" + return markdown.markdown( + text, + extensions=["nl2br"], # 单换行 →
+ ) + + +def _render_html( + day_str: str, + stats: dict, + articles: list[dict], + high_events: list[dict], + hi_threshold: int, + sentiments: Counter, + importances: Counter, + event_types: Counter, + sources: Counter, + ai_summary: str, +) -> str: + """组装完整 HTML。""" + + # AI 摘要 Markdown → HTML + summary_html = _md_to_html(ai_summary) if ai_summary.strip() else "

暂无 AI 摘要

" + + # 事件表格 + events_table = _render_event_table(high_events) + + # 情绪 + pos = sentiments.get("positive", 0) + neg = sentiments.get("negative", 0) + neu = sentiments.get("neutral", 0) + total_s = max(pos + neg + neu, 1) + sentiment_section = ( + f'
' + f'
' + f'
' + f'
' + f'
' + f'
' + f'🟢 利好 {pos} ({pos/total_s:.0%})' + f'🔴 利空 {neg} ({neg/total_s:.0%})' + f'⚪ 中性 {neu} ({neu/total_s:.0%})' + f'
' + ) + + # 重要度 + imp_rows = "".join( + f"等级 {k}{v}" + for k, v in sorted(importances.items()) + ) + importance_table = f"{imp_rows}
重要度数量
" + + # 事件类型 + et_rows = "".join( + f"{k}{v}" + for k, v in event_types.most_common(10) + ) + event_type_table = f"{et_rows}
事件类型数量
" + + # 文章来源 + src_rows = "".join( + f"{_source_name(k)}{v}" + for k, v in sources.most_common(15) + ) + source_table = f"{src_rows}
来源文章数
" + + return _HTML_TEMPLATE.format( + date=day_str, + generated_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + ai_summary=summary_html, + hi_threshold=hi_threshold, + high_count=len(high_events), + events_table=events_table, + raw_total=stats["raw_total"], + proc=stats["proc"], + deduped=stats["deduped"], + emb_count=stats["emb_count"], + qdrant_count=stats["qdrant_count"], + sentiment_section=sentiment_section, + importance_table=importance_table, + event_type_table=event_type_table, + source_table=source_table, + ) + + +def _dedup_events(events: list[dict]) -> list[dict]: + """事件级去重:同一 URL + 同一中文标题的事件合并。 + + 合并策略: + - stock_codes 取并集 + - importance / sentiment / summary_zh / event_type 保留 importance 更高的一条 + - 按 importance 降序排列 + + Args: + events: 含 article 属性的事件列表 + + Returns: + 去重合并后的事件列表 + """ + groups: dict[tuple[str, str], dict] = {} + for ev in events: + article = ev.get("article", {}) + url = article.get("url", "") + title_zh = article.get("title_zh", "") or article.get("title", "") + key = (url, title_zh) + + if key in groups: + existing = groups[key] + # 合并 stock_codes + existing_codes = set(existing.get("stock_codes", [])) + new_codes = set(ev.get("stock_codes", [])) + existing["stock_codes"] = sorted(existing_codes | new_codes) + # 保留 importance 更高的事件详情 + if ev.get("importance", 0) > existing.get("importance", 0): + for field in ("importance", "sentiment", "summary_zh", "event_type"): + if field in ev: + existing[field] = ev[field] + elif ev.get("importance", 0) == existing.get("importance", 0): + # 同等重要度:保留更长的 summary_zh(信息量更大) + if len(ev.get("summary_zh", "")) > len(existing.get("summary_zh", "")): + existing["summary_zh"] = ev["summary_zh"] + existing["event_type"] = ev.get("event_type", existing.get("event_type", "")) + else: + groups[key] = { + "importance": ev.get("importance", 0), + "sentiment": ev.get("sentiment", ""), + "summary_zh": ev.get("summary_zh", ""), + "event_type": ev.get("event_type", ""), + "stock_codes": sorted(ev.get("stock_codes", [])), + "article": ev.get("article", {}), + } + + return sorted(groups.values(), key=lambda e: -e.get("importance", 0)) + + +def _render_event_table(events: list[dict]) -> str: + """渲染事件表格。""" + if not events: + return "

暂无符合条件的数据

" + + rows: list[str] = [] + for i, ev in enumerate(events, 1): + sentiment = ev.get("sentiment", "") + icon = {"positive": "🟢", "negative": "🔴", "neutral": "⚪"}.get(sentiment, "") + badge_cls = {"positive": "badge-pos", "negative": "badge-neg"}.get( + sentiment, "badge-neu" + ) + imp = ev.get("importance", 0) + imp_cls = f"imp-{imp}" if imp >= 4 else "" + + article = ev.get("article", {}) + title = article.get("title_zh") or article.get("title", "")[:80] + url = article.get("url", "") + codes = ",".join(ev.get("stock_codes", [])[:5]) + code_str = f" [{codes}]" if codes else "" + # 从 URL 域名推导来源名称,确保链接域名与来源标注一致 + src_name = _url_source_label(url, article.get("source_id", "")) + + cols = [ + f"{i}", + f'{icon}', + f'{title}{code_str}', + f'{imp}', + f"{ev.get('event_type', '')}", + f"{(ev.get('summary_zh', '') or '')[:80]} [{src_name}]", + ] + rows.append(f'{"".join(cols)}') + + headers = ["#", "", "标题", "重要度", "事件类型", "摘要"] + header_row = "".join(f"{h}" for h in headers) + return f"{header_row}{''.join(rows)}
" diff --git a/scripts/_load_config.sh b/scripts/_load_config.sh new file mode 100644 index 0000000..1c3659d --- /dev/null +++ b/scripts/_load_config.sh @@ -0,0 +1,46 @@ +# ============================================= +# 被其他脚本 source,从 system.yaml 导出环境变量 +# ============================================= +# 用法:source scripts/_load_config.sh +# ============================================= + +_load_yaml_value() { + # 从 system.yaml 读取一个值 + # 用法: _load_yaml_value "servers.overseas_host" + local key_path="$1" + local default="${2:-}" + + if command -v python3 &>/dev/null; then + local value + value=$(python3 -c " +import yaml, sys +try: + with open('configs/system.yaml') as f: + cfg = yaml.safe_load(f) + parts = '${key_path}'.split('.') + val = cfg + for p in parts: + val = val[p] + print(val) +except Exception: + sys.exit(1) +" 2>/dev/null) && echo "$value" && return + fi + echo "$default" +} + +# ── 导出脚本需要的变量 ────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONFIG_DIR="$SCRIPT_DIR/../configs" + +if [ -f "$CONFIG_DIR/system.yaml" ]; then + export CRAWL_PROJECT_DIR="$(_load_yaml_value "servers.overseas_path" "/opt/intlgrab")" + export LOCAL_PROJECT_DIR="$(_load_yaml_value "servers.domestic_path" "/home/pi/intlnews")" + export OVERSEAS_HOST="$(_load_yaml_value "servers.overseas_host" "")" + export SYNC_SENTINEL="$(_load_yaml_value "sync.sentinel" "/tmp/en_news_sync_done")" + export SYNC_PACK_DIR="$(_load_yaml_value "sync.pack_dir" "/tmp")" +else + echo "WARNING: configs/system.yaml not found, using defaults" + export CRAWL_PROJECT_DIR="/opt/intlgrab" + export LOCAL_PROJECT_DIR="/home/pi/intlnews" +fi diff --git a/scripts/cleanup_logs.sh b/scripts/cleanup_logs.sh new file mode 100755 index 0000000..527937b --- /dev/null +++ b/scripts/cleanup_logs.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# ============================================= +# 日志清理:删除 14 天前的日志文件 +# 适用于海外 + 国内服务器 +# ============================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +LOG_DIR="$PROJECT_DIR/logs" +RETENTION_DAYS=14 + +if [ ! -d "$LOG_DIR" ]; then + exit 0 +fi + +DELETED=$(find "$LOG_DIR" -type f -name "*.log" -mtime +$RETENTION_DAYS 2>/dev/null | wc -l) + +if [ "$DELETED" -gt 0 ]; then + find "$LOG_DIR" -type f -name "*.log" -mtime +$RETENTION_DAYS -delete 2>/dev/null + echo "[$(date)] 清理完成: 删除 $DELETED 个旧日志 (>${RETENTION_DAYS}d)" +else + echo "[$(date)] 无旧日志需要清理" +fi diff --git a/scripts/domestic_crawl_2g.sh b/scripts/domestic_crawl_2g.sh new file mode 100755 index 0000000..88a9e9f --- /dev/null +++ b/scripts/domestic_crawl_2g.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# ============================================= +# 国内服务器:2G headless stealth 抓取 +# ============================================= +# 适用场景: +# - 日常轻量补充抓取 +# - RSS 优先 + headless stealth Web 回退 +# - 可选 SOCKS5 代理(修改 profiles/2g_headless.yaml 中 proxy.enabled) +# +# 用法: +# bash scripts/domestic_crawl_2g.sh # 抓取全部源 +# bash scripts/domestic_crawl_2g.sh reuters # 只抓取指定源 +# ============================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +cd "$PROJECT_DIR" + +LOG() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } + +LOG "══════ 国内 2G headless stealth 抓取 ══════" +LOG "Profile: 2g_headless" +LOG "内存上限: 1800 MB" +LOG "浏览器模式: stealth headless" + +# ── 加载 .env ── +export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true) + +# ── 设置 Profile ── +export EN_NEWS_PROFILE="2g_headless" + +# ── 执行抓取 ── +SOURCE_ARG="" +if [ $# -ge 1 ]; then + SOURCE_ARG="--source $1" + LOG "目标源: $1" +else + LOG "目标源: 全部启用源" +fi + +PYTHONPATH=. .venv/bin/python3 -c " +from crawler.orchestrator import run_crawl_sync +import sys +source_filter = sys.argv[1] if len(sys.argv) > 1 else None +stats = run_crawl_sync(source_filter=source_filter) +print(f'抓取完成: {stats.sources_crawled} 源, {stats.total_articles} 篇') +" "$1" + +LOG "══════ 2G 抓取完成 ✅ ══════" diff --git a/scripts/domestic_crawl_8g.sh b/scripts/domestic_crawl_8g.sh new file mode 100755 index 0000000..0f3aa8c --- /dev/null +++ b/scripts/domestic_crawl_8g.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# ============================================= +# 国内服务器:8G headful + HTTP 代理抓取 +# ============================================= +# 适用场景: +# - 所有新闻源直接使用 headful Playwright 浏览器抓取 +# - HTTP 代理 (127.0.0.1:3128) → Privoxy → SOCKS5(ss-local) 访问海外 +# - 比海外 RSS 方式抓取更可靠,解决反爬问题 +# +# 前置条件: +# sudo apt install xvfb +# Xvfb :99 -screen 0 1280x1024x24 & # 首次启动 +# HTTP 代理 127.0.0.1:3128 已运行(privoxy) +# +# 用法: +# bash scripts/domestic_crawl_8g.sh # 抓取全部源 +# bash scripts/domestic_crawl_8g.sh reuters # 只抓取指定源 +# ============================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +cd "$PROJECT_DIR" + +LOG() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } + +LOG "══════ 国内 8G headful + HTTP 代理抓取 ══════" +LOG "Profile: 8g_headful" +LOG "内存上限: 7500 MB" +LOG "浏览器模式: headful (Xvfb :99)" +LOG "HTTP 代理: 127.0.0.1:3128 (Privoxy → SOCKS5)" + +# ── 加载 .env ── +export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true) + +# ── 启动 Xvfb(如果尚未运行)── +if ! pgrep -x "Xvfb" > /dev/null; then + LOG "启动 Xvfb 虚拟显示器 :99 ..." + Xvfb :99 -screen 0 1280x1024x24 -ac +extension RANDR & + sleep 1 + LOG "Xvfb 已启动 (PID: $(pgrep -x Xvfb))" +else + LOG "Xvfb 已在运行 (PID: $(pgrep -x Xvfb))" +fi + +export DISPLAY=:99 + +# ── 设置 Profile ── +export EN_NEWS_PROFILE="8g_headful" + +# ── 执行抓取 ── +SOURCE_ARG="" +if [ $# -ge 1 ]; then + SOURCE_ARG="--source $1" + LOG "目标源: $1" +else + LOG "目标源: 全部启用源" +fi + +PYTHONPATH=. .venv/bin/python3 -c " +from crawler.orchestrator import run_crawl_sync +import sys +source_filter = sys.argv[1] if len(sys.argv) > 1 else None +stats = run_crawl_sync(source_filter=source_filter) +print(f'抓取完成: {stats.sources_crawled} 源, {stats.total_articles} 篇') +" "${1:-}" + +LOG "══════ 8G 抓取完成 ✅ ══════" diff --git a/scripts/domestic_full.sh b/scripts/domestic_full.sh new file mode 100755 index 0000000..ae4b06f --- /dev/null +++ b/scripts/domestic_full.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# ============================================= +# 国内服务器:全流程自动化(独立运行,无需海外) +# 1. M1 Pi 抓取(headful Playwright + HTTP 代理) +# 2. 全链路 M2→M6(含日报) +# ============================================= +# 每天 06:00 / 12:00 / 18:00 / 22:00 各执行一次 +# ============================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +cd "$PROJECT_DIR" + +LOG() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } + +LOG "══════ 国内全流程开始 ══════" + +# ── 加载 .env ── +export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true) + +# ── 1. M1 Pi 抓取(headful Playwright + HTTP 代理)── +LOG "[1/2] Pi M1 抓取(8G headful + HTTP 代理)..." +bash "$SCRIPT_DIR/domestic_crawl_8g.sh" 2>&1 | tail -5 || LOG "WARNING: 部分源抓取失败,继续管道" + +# ── 2. M2→M6 管道(含日报)── +LOG "[2/2] 全链路管道..." +bash "$SCRIPT_DIR/pipeline.sh" 2>&1 | tail -10 + +LOG "══════ 国内全流程完成 ✅ ══════" diff --git a/scripts/domestic_sync.sh b/scripts/domestic_sync.sh new file mode 100644 index 0000000..4523c23 --- /dev/null +++ b/scripts/domestic_sync.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# ============================================= +# 国内服务器:从海外拉取数据并合并到本地 data/raw/ +# ============================================= +# 幂等:多次执行不会产生重复数据 +# 用法:./scripts/domestic_sync.sh [日期] +# ============================================= +set -euo pipefail + +# 从 system.yaml 读取配置 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/_load_config.sh" + +LOCAL_DATA="$LOCAL_PROJECT_DIR/data/raw" +OVERSEAS_RAW="$CRAWL_PROJECT_DIR/data/raw" +PACK_TMP="${SYNC_PACK_DIR:-/tmp}" + +# ── 确定新闻日 ────────────────────────────────────── +if [ $# -ge 1 ]; then + NEWS_DAY="$1" +else + CUTOFF=6 + HOUR=$(date +%H) + if [ "$HOUR" -lt "$CUTOFF" ]; then + NEWS_DAY=$(date -d "yesterday" +%Y%m%d) + else + NEWS_DAY=$(date +%Y%m%d) + fi +fi + +PACK_FILE="en_news_${NEWS_DAY}.tar.gz" + +echo "[$(date)] ═══ 国内同步开始,新闻日: $NEWS_DAY ═══" + +# ── 1. 增量 rsync ───────────────────────────────── +echo "[$(date)] [1/4] rsync 增量同步 data/raw/ ..." +mkdir -p "$LOCAL_DATA" +rsync -avz --ignore-existing \ + -e "ssh -o ConnectTimeout=10" \ + "$OVERSEAS_HOST:$OVERSEAS_RAW/" \ + "$LOCAL_DATA/" \ + 2>&1 | tail -3 + +# ── 2. 拉取压缩包 ───────────────────────────────── +echo "[$(date)] [2/4] 尝试拉取压缩包 ..." +PACK_LOCAL="$PACK_TMP/$PACK_FILE" + +if ssh -o ConnectTimeout=5 "$OVERSEAS_HOST" "[ -f $OVERSEAS_RAW/$PACK_FILE ]" 2>/dev/null; then + scp "$OVERSEAS_HOST:$OVERSEAS_RAW/$PACK_FILE" "$PACK_LOCAL" 2>/dev/null + echo "[$(date)] 压缩包拉取成功: $PACK_LOCAL ($(du -h "$PACK_LOCAL" | cut -f1))" +elif ssh -o ConnectTimeout=5 "$OVERSEAS_HOST" "[ -f /tmp/$PACK_FILE ]" 2>/dev/null; then + scp "$OVERSEAS_HOST:/tmp/$PACK_FILE" "$PACK_LOCAL" 2>/dev/null + echo "[$(date)] 压缩包拉取成功 (海外 /tmp): $PACK_LOCAL ($(du -h "$PACK_LOCAL" | cut -f1))" +else + echo "[$(date)] 海外无压缩包,跳过" +fi + +# ── 3. 解压合并 ─────────────────────────────────── +if [ -f "$PACK_LOCAL" ]; then + echo "[$(date)] [3/4] 解压合并到 $LOCAL_DATA ..." + tar xzf "$PACK_LOCAL" -C "$LOCAL_DATA/" --overwrite 2>&1 + echo "[$(date)] 解压完成" + mkdir -p "$LOCAL_PROJECT_DIR/data/archive" + mv "$PACK_LOCAL" "$LOCAL_PROJECT_DIR/data/archive/$PACK_FILE" 2>/dev/null || true +else + echo "[$(date)] [3/4] 无压缩包,跳过解压" +fi + +# ── 4. 验证 ─────────────────────────────────────── +echo "[$(date)] [4/4] 验证..." +echo "[$(date)] 文件数: $(find "$LOCAL_DATA" -type f ! -name '*.tar.gz' | wc -l)" +echo "[$(date)] 总大小: $(du -sh "$LOCAL_DATA" 2>/dev/null | cut -f1)" +echo "[$(date)] ═══ 国内同步完成 ✅ ═══" diff --git a/scripts/fix_publish_time.sh b/scripts/fix_publish_time.sh new file mode 100755 index 0000000..da9cef5 --- /dev/null +++ b/scripts/fix_publish_time.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# ============================================= +# 精准修复: 为 publish_time 为空的存量数据补日期 +# ============================================= +# 场景: extractor 升级后,已处理的文章 publish_time 仍为空 +# 策略: 重新提取日期 → 更新 processed JSON → 更新 events JSON +# 影响: 仅修改 JSON 文件的 publish_time 字段,不触发 LLM 重跑 +# ============================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +cd "$PROJECT_DIR" + +echo "[$(date)] ═══ publish_time 精准修复开始 ═══" + +.venv/bin/python3 << 'PYEOF' +import json +import logging +from pathlib import Path + +logging.basicConfig( + level=logging.INFO, + format="%(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + +# ── 复用 extractor 的日期提取(不依赖 HTML 文件存在) ── + +def extract_date_from_url(url: str) -> str: + """从 URL 提取日期。""" + import re + from datetime import datetime + url_patterns = [ + r"/(\d{4})/(\d{2})/(\d{2})/", + r"/(\d{4})(\d{2})(\d{2})/", + r"-(\d{4})(\d{2})(\d{2})(?:[/-]|$)", + r"-(\d{4})-(\d{2})-(\d{2})[/-]", + ] + for pat in url_patterns: + m = re.search(pat, url) + if m: + try: + y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3)) + return datetime(y, mo, d).isoformat() + except ValueError: + continue + return "" + + +def extract_date_from_html(html_path: str, url: str) -> str: + """从 HTML 文件提取日期。""" + from extractor.extractor import _extract_publish_time + p = Path(html_path) + if p.exists(): + try: + html = p.read_text(encoding="utf-8") + return _extract_publish_time(html, url=url) or "" + except Exception: + pass + return "" + + +def extract_date_from_md(md_path: str, url: str) -> str: + """从 Markdown 文件提取日期(RSS 源在 MD 中写入发布时间)。""" + import re + p = Path(md_path) + if not p.exists(): + return "" + try: + md = p.read_text(encoding="utf-8") + # RSS 格式: **发布时间**: 2026-06-19T14:30:00 + m = re.search(r"\*\*发布时间\*\*:?\s*([^\n]+)", md) + if m: + return m.group(1).strip() + except Exception: + pass + return extract_date_from_url(url) + + +# ════════════════════════════════════════════ +# 阶段 1: 修复 data/processed/ 中的 publish_time +# ════════════════════════════════════════════ + +logger.info("═══ 阶段 1: 扫描 data/processed/ ═══") +fixed_processed = 0 +skipped_ok = 0 +skipped_no_source = 0 + +for proc_file in Path("data/processed").glob("*/*/*.json"): + if proc_file.name == "index.jsonl": + continue + try: + data = json.loads(proc_file.read_text(encoding="utf-8")) + except Exception: + continue + + # 只处理 publish_time 为空的 + pt = (data.get("publish_time") or "").strip() + if pt: + skipped_ok += 1 + continue + + url = data.get("url", "") + html_path = data.get("html_path", "") + md_path = data.get("md_path", "") + + # 尝试提取日期 + new_pt = "" + if html_path: + new_pt = extract_date_from_html(html_path, url) + if not new_pt and md_path: + new_pt = extract_date_from_md(md_path, url) + if not new_pt: + new_pt = extract_date_from_url(url) + + if new_pt: + data["publish_time"] = new_pt + proc_file.write_text( + json.dumps(data, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + fixed_processed += 1 + logger.debug("processed: %s → %s", proc_file.name, new_pt) + else: + skipped_no_source += 1 + +logger.info( + "阶段 1 完成: 修复 %d, 已有日期 %d, 仍无法提取 %d", + fixed_processed, skipped_ok, skipped_no_source, +) + + +# ════════════════════════════════════════════ +# 阶段 2: 同步修复 data/events/ 中的 publish_time +# ════════════════════════════════════════════ + +logger.info("═══ 阶段 2: 扫描 data/events/ ═══") +fixed_events = 0 +skipped_events_ok = 0 +skipped_events_no_url = 0 + +for ev_file in Path("data/events").glob("*/*.json"): + if ev_file.name == "index.json": + continue + try: + data = json.loads(ev_file.read_text(encoding="utf-8")) + except Exception: + continue + + pt = (data.get("publish_time") or "").strip() + if pt: + skipped_events_ok += 1 + continue + + url = data.get("url", "") + new_pt = extract_date_from_url(url) + + if new_pt: + data["publish_time"] = new_pt + ev_file.write_text( + json.dumps(data, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + fixed_events += 1 + logger.debug("events: %s → %s", ev_file.name, new_pt) + else: + skipped_events_no_url += 1 + +logger.info( + "阶段 2 完成: 修复 %d, 已有日期 %d, 仍无法提取 %d", + fixed_events, skipped_events_ok, skipped_events_no_url, +) + +# ── 汇总 ── +total_fixed = fixed_processed + fixed_events +total_still_empty = skipped_no_source + skipped_events_no_url +logger.info("══════ 汇总 ══════") +logger.info("processes 修复: %d, events 修复: %d", fixed_processed, fixed_events) +logger.info("总计修复: %d, 仍为空: %d", total_fixed, total_still_empty) +PYEOF + +echo "[$(date)] ═══ publish_time 精准修复完成 ✅ ═══" diff --git a/scripts/overseas_crawl.sh b/scripts/overseas_crawl.sh new file mode 100755 index 0000000..2c6cf95 --- /dev/null +++ b/scripts/overseas_crawl.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# ============================================= +# 海外服务器:抓取英文财经新闻 +# ============================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +cd "$PROJECT_DIR" + +echo "[$(date)] ═══ 海外抓取开始 ═══" + +PYTHONPATH=. .venv/bin/python3 -c " +from crawler.orchestrator import run_crawl_sync +stats = run_crawl_sync() +print(f'抓取完成: {stats.sources_crawled} 源, {stats.total_articles} 篇') +" + +echo "[$(date)] ═══ 海外抓取完成 ✅ ═══" diff --git a/scripts/overseas_pack.sh b/scripts/overseas_pack.sh new file mode 100644 index 0000000..57c53fc --- /dev/null +++ b/scripts/overseas_pack.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# ============================================= +# 海外服务器:打包压缩当日 data/raw/ 下的所有文件 +# ============================================= +# 用法:./scripts/overseas_pack.sh [日期] +# 不传日期则使用当前新闻日(基于 day_cutoff_hour) +# ============================================= +set -euo pipefail + +# 从 system.yaml 读取配置(有默认值兜底) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/_load_config.sh" + +PROJECT_DIR="${CRAWL_PROJECT_DIR}" +DATA_DIR="$PROJECT_DIR/data/raw" +PACK_TMP="${SYNC_PACK_DIR:-/tmp}" +SENTINEL_FILE="${SYNC_SENTINEL}" + +# ── 确定新闻日 ────────────────────────────────────── +if [ $# -ge 1 ]; then + NEWS_DAY="$1" +else + # 从 system.yaml 读取 cutoff hour + CUTOFF=$(python3 -c " +import yaml +with open('$PROJECT_DIR/configs/system.yaml') as f: + cfg = yaml.safe_load(f) +print(cfg.get('schedule', {}).get('day_cutoff_hour', 6)) +" 2>/dev/null || echo 6) + + HOUR=$(date +%H) + if [ "$HOUR" -lt "$CUTOFF" ]; then + NEWS_DAY=$(date -d "yesterday" +%Y%m%d) + else + NEWS_DAY=$(date +%Y%m%d) + fi +fi + +echo "[$(date)] 新闻日: $NEWS_DAY" + +# ── 检查数据是否存在 ───────────────────────────────── +if [ ! -d "$DATA_DIR" ]; then + echo "[$(date)] WARNING: data/raw/ 不存在,跳过打包" + exit 0 +fi + +# 计算今天目录下的总文件数(排除 tar.gz 和 sentinel) +TOTAL_FILES=$(find "$DATA_DIR" -path "*/$NEWS_DAY/*" -type f ! -name "*.tar.gz" 2>/dev/null | wc -l) +if [ "$TOTAL_FILES" -eq 0 ]; then + echo "[$(date)] WARNING: 新闻日 $NEWS_DAY 无数据文件,跳过打包" + exit 0 +fi + +# ── 打包压缩 ───────────────────────────────────────── +PACK_FILE="$PACK_TMP/en_news_${NEWS_DAY}.tar.gz" + +echo "[$(date)] 打包 $TOTAL_FILES 个文件 → $PACK_FILE ..." + +# 收集当天的所有数据文件(包括任意源下面的日期目录) +find "$DATA_DIR" -path "*/$NEWS_DAY/*" -type f ! -name "*.tar.gz" 2>/dev/null \ + | tar czf "$PACK_FILE" -T - --transform='s|.*/data/raw/||' 2>/dev/null + +PACK_SIZE=$(du -h "$PACK_FILE" | cut -f1) +echo "[$(date)] 打包完成: $PACK_FILE ($PACK_SIZE)" + +# ── 生成哨兵文件 ───────────────────────────────────── +echo "$NEWS_DAY $(date -Iseconds) $PACK_SIZE ($TOTAL_FILES files)" > "$SENTINEL_FILE" +echo "[$(date)] 哨兵文件已更新: $SENTINEL_FILE" + +echo "[$(date)] ✅ 海外打包完成" diff --git a/scripts/pipeline.sh b/scripts/pipeline.sh new file mode 100755 index 0000000..9cf3fa3 --- /dev/null +++ b/scripts/pipeline.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# ============================================= +# 国内服务器:全链路管道 M2 → M3 → M4 → M5 → M6 → 日报 +# ============================================= +# 用法:./scripts/pipeline.sh +# 前提:domestic_sync.sh 已完成 +# ============================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +cd "$PROJECT_DIR" + +echo "[$(date)] ═══ 全链路管道开始 ═══" + +# 加载 .env +export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true) + +# ── M2: 正文提取 ── +echo "[$(date)] [M2] 正文提取..." +.venv/bin/python3 -c " +from extractor.pipeline import process_all_sources +stats = process_all_sources() +print(f'M2: {stats[\"total_articles\"]} 篇, {stats[\"elapsed_sec\"]:.0f}s') +" + +# ── M3: 去重 ── +echo "[$(date)] [M3] 三层去重..." +.venv/bin/python3 -c " +from dedup.pipeline import dedup_all_sources +stats = dedup_all_sources() +print(f'M3: 唯一 {stats[\"unique\"]}/重复 {stats[\"duplicate\"]}, {stats[\"elapsed_sec\"]:.0f}s') +" + +# ── M4: 翻译+事件 ── +echo "[$(date)] [M4] 翻译+事件抽取..." +.venv/bin/python3 -c " +from llm.pipeline import translate_all_deduped +stats = translate_all_deduped() +print(f'M4: {stats[\"success\"]}/{stats[\"total\"]} 篇, {stats[\"elapsed_sec\"]:.0f}s') +" + +# ── M5: 向量生成 ── +echo "[$(date)] [M5] 向量生成..." +.venv/bin/python3 -c " +from embedding.pipeline import embed_all_events +stats = embed_all_events() +print(f'M5: {stats[\"success\"]}/{stats[\"total\"]} 篇, {stats[\"elapsed_sec\"]:.0f}s') +" + +# ── M6: Qdrant 入库 ── +echo "[$(date)] [M6] Qdrant 入库..." +.venv/bin/python3 -c " +from vectorstore.pipeline import ingest_all_embeddings +stats = ingest_all_embeddings() +print(f'M6: {stats[\"ingested\"]}/{stats[\"total\"]} 条, {stats[\"elapsed_sec\"]:.0f}s') +" + +# ── 日报 ── +echo "[$(date)] [日报] 生成日报..." +.venv/bin/python3 -c " +from scheduler.reporter import generate_report +path = generate_report() +print(f'日报: {path}') +" + +echo "[$(date)] ═══ 全链路管道完成 ✅ ═══" diff --git a/tests/test_crawler.py b/tests/test_crawler.py new file mode 100644 index 0000000..002d026 --- /dev/null +++ b/tests/test_crawler.py @@ -0,0 +1,266 @@ +"""M1 爬虫模块测试""" + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from crawler.crawler import ( + ARTICLE_DELAY_SEC, + MAX_MEMORY_MB, + SOURCE_TIMEOUT_SEC, + compute_url_hash, + crawl_source, +) +from crawler.loader import get_source_by_id, load_sources +from crawler.models import ArticleItem, CrawlResult, SourceConfig +from crawler.storage import load_index, write_index_jsonl + +# ════════════════════════════════════════════════ +# URL Hash +# ════════════════════════════════════════════════ + +def test_compute_url_hash_consistency(): + """同一 URL 多次计算 hash 一致""" + h1 = compute_url_hash("https://example.com/article/123") + h2 = compute_url_hash("https://example.com/article/123") + assert h1 == h2 + assert len(h1) == 16 + + +def test_compute_url_hash_different(): + """不同 URL 产生不同 hash""" + h1 = compute_url_hash("https://example.com/a") + h2 = compute_url_hash("https://example.com/b") + assert h1 != h2 + + +# ════════════════════════════════════════════════ +# 配置加载 +# ════════════════════════════════════════════════ + +def test_load_sources_from_real_config(): + """从项目真实配置文件加载""" + sources, settings = load_sources() + assert len(sources) == 12 + assert settings["concurrency"] == 5 + + reuters = sources[0] + assert reuters.id == "reuters" + assert reuters.name == "Reuters" + assert reuters.enabled is True + + +def test_get_source_by_id(): + """按 ID 查找源""" + sources, _ = load_sources() + s = get_source_by_id("cnbc", sources) + assert s is not None + assert s.name == "CNBC" + + s = get_source_by_id("nonexistent", sources) + assert s is None + + +def test_load_sources_missing_file(): + """配置文件不存在时抛出异常""" + with pytest.raises(FileNotFoundError): + load_sources(Path("/nonexistent/path.yaml")) + + +def test_source_config_output_dir(): + """SourceConfig.output_dir 属性""" + s = SourceConfig( + id="reuters", + name="Reuters", + homepage="https://example.com", + article_url_pattern="/article/", + ) + today = __import__("datetime").datetime.now().strftime("%Y%m%d") + assert str(s.output_dir) == f"data/raw/reuters/{today}" + + +# ════════════════════════════════════════════════ +# 存储 +# ════════════════════════════════════════════════ + +def test_write_and_load_index_jsonl(tmp_path: Path, monkeypatch): + """写入 index.jsonl 后再读取,数据一致""" + # 临时替换 data/raw 路径 + import crawler.storage as storage_mod + + articles = [ + ArticleItem( + source_id="test_source", + source_name="Test Source", + url=f"https://example.com/article/{i}", + url_hash=f"hash{i:04d}", + title=f"Test Article {i}", + crawl_time="2026-06-21T00:00:00", + html_path=f"data/raw/test_source/20260621/hash{i:04d}.html", + status="success", + ) + for i in range(3) + ] + + result = CrawlResult( + source_id="test_source", + source_name="Test Source", + total_found=3, + total_success=3, + articles=articles, + ) + + # Patch Path to use tmp_path + orig_path = Path + + def mock_path(p: str) -> Path: + p_str = str(p) + if p_str.startswith("data/raw/"): + return orig_path(tmp_path) / p_str + return orig_path(p_str) + + monkeypatch.setattr(storage_mod, "Path", mock_path) + + index_path = write_index_jsonl(result) + assert index_path.exists() + + # 读取 + loaded = load_index("test_source", "20260621") + assert len(loaded) == 3 + assert loaded[0].source_id == "test_source" + assert loaded[0].title == "Test Article 0" + + +def test_load_index_missing_file(): + """不存在的 index 返回空列表""" + articles = load_index("nonexistent", "20990101") + assert articles == [] + + +def test_write_index_jsonl_dedup(tmp_path: Path, monkeypatch): + """重复 url_hash 不重复写入""" + import crawler.storage as storage_mod + + article = ArticleItem( + source_id="dedup_test", + source_name="Dedup Test", + url="https://example.com/same", + url_hash="same_hash_0001", + title="Same Article", + crawl_time="2026-06-21T00:00:00", + html_path="data/raw/dedup_test/20260621/same_hash_0001.html", + status="success", + ) + + result1 = CrawlResult(source_id="dedup_test", source_name="Dedup Test", + total_success=1, articles=[article]) + result2 = CrawlResult(source_id="dedup_test", source_name="Dedup Test", + total_success=1, articles=[article]) + + def mock_path(p: str) -> Path: + p_str = str(p) + if p_str.startswith("data/raw/"): + return Path(tmp_path) / p_str + return Path(p_str) + + monkeypatch.setattr(storage_mod, "Path", mock_path) + + write_index_jsonl(result1) + write_index_jsonl(result2) # 重复写入 + + loaded = load_index("dedup_test", "20260621") + assert len(loaded) == 1 # 去重 + + +# ════════════════════════════════════════════════ +# 爬虫引擎 (Mock) +# ════════════════════════════════════════════════ + +@pytest.mark.asyncio +async def test_crawl_source_with_mock(): + """Mock Crawl4AI,测试 crawl_source 流程""" + source = SourceConfig( + id="mock_source", + name="Mock Source", + homepage="https://mock.example.com/", + article_url_pattern="/news/", + js_render=False, + max_articles_per_run=5, + ) + + # Mock Crawl4AI 的返回 + mock_html = 'Article 1Article 2' + mock_result = MagicMock() + mock_result.success = True + mock_result.html = mock_html + mock_result.markdown = "# Test Article\n\nContent here." + mock_result.metadata = {"title": "Test Article"} + mock_result.error_message = "" + + with patch("crawler.crawler.AsyncWebCrawler") as mock_crawler_cls: + mock_crawler = MagicMock() + mock_crawler.arun = AsyncMock(return_value=mock_result) + mock_crawler_cls.return_value.__aenter__ = AsyncMock(return_value=mock_crawler) + mock_crawler_cls.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await crawl_source(source) + + assert result.source_id == "mock_source" + assert result.total_found == 2 + assert result.total_success == 2 + assert result.total_failed == 0 + assert len(result.articles) == 2 + + # 每条 article 都有正确的 source_id + for article in result.articles: + assert article.source_id == "mock_source" + assert article.status == "success" + assert article.html_path + + +@pytest.mark.asyncio +async def test_crawl_source_homepage_failure(): + """首页抓取失败时优雅降级""" + source = SourceConfig( + id="fail_source", + name="Fail Source", + homepage="https://fail.example.com/", + article_url_pattern="/news/", + js_render=False, + ) + + mock_result = MagicMock() + mock_result.success = False + mock_result.html = "" + mock_result.error_message = "Connection timeout" + + with patch("crawler.crawler.AsyncWebCrawler") as mock_crawler_cls: + mock_crawler = MagicMock() + mock_crawler.arun = AsyncMock(return_value=mock_result) + mock_crawler_cls.return_value.__aenter__ = AsyncMock(return_value=mock_crawler) + mock_crawler_cls.return_value.__aexit__ = AsyncMock(return_value=False) + + result = await crawl_source(source) + + assert result.total_found == 0 + assert result.total_success == 0 + + +# ════════════════════════════════════════════════ +# 资源限制常量 +# ════════════════════════════════════════════════ + +def test_max_memory_mb(): + """内存限制 < 2000 MB""" + assert 0 < MAX_MEMORY_MB < 2000 + + +def test_source_timeout_sec(): + """单源超时在合理范围""" + assert SOURCE_TIMEOUT_SEC >= 3600 # 至少 1 小时 + + +def test_article_delay_sec(): + """文章间隔 ≥ 1 秒""" + assert ARTICLE_DELAY_SEC >= 1.0 diff --git a/tests/test_dedup.py b/tests/test_dedup.py new file mode 100644 index 0000000..e33f380 --- /dev/null +++ b/tests/test_dedup.py @@ -0,0 +1,533 @@ +"""M3 三层去重模块单元测试。""" + +from pathlib import Path + +import pytest + +from dedup import ( + DEFAULT_HAMMING_THRESHOLD, + Deduper, + DedupLayer, + DedupResult, + Fingerprint, + FingerprintStore, + article_to_fingerprint, + content_hash, + hamming, + normalize_content, + simhash64, +) +from extractor.models import ProcessedArticle + +# --------------------------------------------------------------------------- # +# 辅助工厂函数 +# --------------------------------------------------------------------------- # + + +def _make_article( + *, + url: str = "https://www.reuters.com/business/1", + url_hash: str = "abc1234567890000", + source_id: str = "reuters", + source_name: str = "Reuters", + title: str = "Fed Holds Rates Steady as Markets Rally", + content: str = ( + "The Federal Reserve held interest rates steady on Wednesday, " + "citing solid economic growth and a strong labor market. " + "Markets rallied in response, with the S&P 500 gaining 1.2 percent." + ), + publish_time: str = "2026-06-16T10:00:00", + word_count: int = 0, +) -> ProcessedArticle: + return ProcessedArticle( + source_id=source_id, + source_name=source_name, + url=url, + url_hash=url_hash, + title=title, + content=content, + publish_time=publish_time, + word_count=word_count or len(content.split()), + ) + + +@pytest.fixture +def tmp_db(tmp_path: Path) -> Path: + return tmp_path / "fp.sqlite3" + + +# --------------------------------------------------------------------------- # +# hasher 测试 +# --------------------------------------------------------------------------- # + + +class TestNormalizeContent: + """normalize_content 函数测试。""" + + def test_strips_punctuation_and_whitespace(self): + norm = normalize_content("Hello, world!\nThis is text.") + assert norm == "HelloworldThisistext" + + def test_handles_empty(self): + assert normalize_content("") == "" + assert normalize_content(" \n\t ") == "" + + def test_preserves_letters_and_digits(self): + # $ 是货币符号(Sc 类别),会被保留;% 和 . 是标点(Po)会被移除 + norm = normalize_content("AAPL up 5.2% to $150.00!") + assert norm == "AAPLup52to$15000" + + def test_handles_chinese_characters(self): + norm = normalize_content("宁德时代 发布 新一代 麒麟电池!") + assert norm == "宁德时代发布新一代麒麟电池" + + +class TestContentHash: + """content_hash 函数测试。""" + + def test_deterministic(self): + a = "The Fed raised rates today." + b = "The Fed raised rates today." + assert content_hash(a) == content_hash(b) + + def test_punctuation_invariant(self): + a = "Apple reports record earnings." + b = "Apple, reports... record earnings!!!" + assert content_hash(a) == content_hash(b) + + def test_differs_for_different_text(self): + assert content_hash("Fed raises rates") != content_hash("Fed cuts rates") + + +class TestSimhash64: + """simhash64 函数测试。""" + + def test_identical_text_same_value(self): + text = "The Federal Reserve held interest rates steady on Wednesday." + assert simhash64(text) == simhash64(text) + + def test_minor_changes_close_distance(self): + """轻量改写,长文本汉明距离应在阈值内。""" + base = ( + "The Federal Reserve held interest rates steady on Wednesday, " + "citing solid economic growth and a strong labor market. " + "Markets rallied in response, with the S&P 500 gaining 1.2 percent. " + "Analysts expect rates to remain unchanged through the summer." + ) * 2 + rewritten = "WASHINGTON (Reuters) - " + base + " (Reporting by John Smith)" + d = hamming(simhash64(base), simhash64(rewritten)) + assert d <= DEFAULT_HAMMING_THRESHOLD, ( + f"长文本前后加来源标识汉明距离 {d} 不应超过阈值" + ) + + def test_unrelated_text_far_distance(self): + """完全不相关的两段长文本汉明距离应远大于阈值。""" + a = "The Federal Reserve held interest rates steady on Wednesday." * 5 + b = "Apple announced a new iPhone model with revolutionary features." * 5 + d = hamming(simhash64(a), simhash64(b)) + assert d > DEFAULT_HAMMING_THRESHOLD * 2 + + def test_empty_returns_zero(self): + assert simhash64("") == 0 + assert simhash64(" ") == 0 + + +class TestHamming: + """hamming 距离函数测试。""" + + def test_same_value_zero(self): + assert hamming(0, 0) == 0 + assert hamming(0xDEADBEEF, 0xDEADBEEF) == 0 + + def test_basic(self): + assert hamming(0xFF, 0x00) == 8 + assert hamming(0xFF00FF00, 0x00FF00FF) == 32 + + def test_single_bit(self): + assert hamming(1, 0) == 1 + assert hamming(1 << 63, 0) == 1 + + +# --------------------------------------------------------------------------- # +# FingerprintStore 测试 +# --------------------------------------------------------------------------- # + + +class TestFingerprintStore: + """FingerprintStore 功能测试。""" + + def test_upsert_and_get(self, tmp_db): + fp = Fingerprint( + url_hash="hash1", + content_hash="ch1", + simhash=0xDEADBEEFCAFEBABE, + source_id="reuters", + url="https://x/1", + title="Test Title", + publish_date="2026-06-16", + ) + with FingerprintStore(tmp_db) as store: + store.upsert(fp) + got = store.get_by_url_hash("hash1") + assert got is not None + assert got.content_hash == "ch1" + assert got.simhash == 0xDEADBEEFCAFEBABE + assert got.publish_date == "2026-06-16" + + def test_upsert_replaces_existing(self, tmp_db): + base = Fingerprint( + url_hash="h", + content_hash="ch1", + simhash=1, + source_id="reuters", + url="u", + title="t", + ) + updated = base.model_copy(update={"content_hash": "ch2", "simhash": 999}) + with FingerprintStore(tmp_db) as store: + store.upsert(base) + store.upsert(updated) + got = store.get_by_url_hash("h") + assert got is not None + assert got.content_hash == "ch2" + assert got.simhash == 999 + assert store.count() == 1 + + def test_find_by_content_hash(self, tmp_db): + with FingerprintStore(tmp_db) as store: + store.upsert(Fingerprint( + url_hash="h1", content_hash="ch", simhash=0, + source_id="reuters", url="u1", title="t1", + )) + assert store.find_by_content_hash("ch") is not None + assert store.find_by_content_hash("nope") is None + + def test_candidates_within_window(self, tmp_db): + with FingerprintStore(tmp_db) as store: + for d, h in [ + ("2026-05-01", "old"), + ("2026-06-15", "near"), + ("2026-07-30", "far"), + ]: + store.upsert(Fingerprint( + url_hash=h, content_hash=h, simhash=0, + source_id="reuters", url=f"u/{h}", title=h, publish_date=d, + )) + cands = store.candidates_for_simhash("2026-06-16", window_days=7) + url_hashes = sorted(c.url_hash for c in cands) + assert url_hashes == ["near"] + + def test_candidates_no_date_returns_all(self, tmp_db): + with FingerprintStore(tmp_db) as store: + store.upsert(Fingerprint( + url_hash="h1", content_hash="c1", simhash=0, + source_id="reuters", url="u", title="t", publish_date=None, + )) + cands = store.candidates_for_simhash(None, 30) + assert len(cands) == 1 + + def test_simhash_high_bit_hex(self, tmp_db): + """64 位 SimHash 高位为 1 时,hex 存取应保持无符号。""" + high = (1 << 63) | 0x1234 + with FingerprintStore(tmp_db) as store: + store.upsert(Fingerprint( + url_hash="h", content_hash="c", simhash=high, + source_id="reuters", url="u", title="t", + )) + got = store.get_by_url_hash("h") + assert got is not None + assert got.simhash == high + + def test_count_by_source(self, tmp_db): + with FingerprintStore(tmp_db) as store: + for i, src in enumerate(["reuters", "reuters", "cnbc"]): + store.upsert(Fingerprint( + url_hash=f"h{i}", content_hash=f"c{i}", simhash=i, + source_id=src, url=f"u{i}", title=f"t{i}", + )) + counts = store.count_by_source() + assert counts == {"reuters": 2, "cnbc": 1} + + def test_date_range(self, tmp_db): + with FingerprintStore(tmp_db) as store: + store.upsert(Fingerprint( + url_hash="h1", content_hash="c1", simhash=0, + source_id="reuters", url="u1", title="t1", + publish_date="2026-06-10", + )) + store.upsert(Fingerprint( + url_hash="h2", content_hash="c2", simhash=0, + source_id="cnbc", url="u2", title="t2", + publish_date="2026-06-20", + )) + lo, hi = store.date_range() + assert lo == "2026-06-10" + assert hi == "2026-06-20" + + def test_delete(self, tmp_db): + with FingerprintStore(tmp_db) as store: + store.upsert(Fingerprint( + url_hash="h1", content_hash="c1", simhash=0, + source_id="reuters", url="u1", title="t1", + )) + store.delete("h1") + assert store.get_by_url_hash("h1") is None + assert store.count() == 0 + + +# --------------------------------------------------------------------------- # +# article_to_fingerprint 测试 +# --------------------------------------------------------------------------- # + + +class TestArticleToFingerprint: + """article_to_fingerprint 转换测试。""" + + def test_fields(self): + art = _make_article() + fp = article_to_fingerprint(art) + assert fp.url_hash == art.url_hash + assert fp.simhash == simhash64(art.content) + assert fp.content_hash == content_hash(art.content) + assert fp.publish_date == "2026-06-16" + assert fp.source_id == "reuters" + + def test_handles_empty_publish_time(self): + art = _make_article(publish_time="") + fp = article_to_fingerprint(art) + assert fp.publish_date is None + + +# --------------------------------------------------------------------------- # +# Deduper 三层去重测试 +# --------------------------------------------------------------------------- # + + +class TestDeduper: + """Deduper 三层去重功能测试。""" + + def test_first_article_is_unique(self, tmp_db): + art = _make_article() + with Deduper(db_path=tmp_db) as d: + result = d.ingest(art) + assert not result.is_duplicate + assert result.matched_layer is None + assert d.stats().total == 1 + + def test_layer1_url_hash(self, tmp_db): + """同一 url_hash 直接命中 L1。""" + a1 = _make_article() + a2 = _make_article() # 同 url_hash 同 url + with Deduper(db_path=tmp_db) as d: + d.ingest(a1) + result = d.ingest(a2) + assert result.is_duplicate + assert result.matched_layer == DedupLayer.URL + assert d.stats().total == 1, "L1 命中应不写入新指纹" + + def test_layer2_content_hash(self, tmp_db): + """url 不同但 content 完全一致 → L2。""" + a1 = _make_article(url="https://a.com/1", url_hash="hash1aaaaaaaaaaa") + a2 = _make_article(url="https://b.com/2", url_hash="hash2bbbbbbbbbbb") + with Deduper(db_path=tmp_db) as d: + d.ingest(a1) + result = d.ingest(a2) + assert result.is_duplicate + assert result.matched_layer == DedupLayer.CONTENT + assert result.matched_url_hash == "hash1aaaaaaaaaaa" + + def test_layer2_punctuation_difference_caught(self, tmp_db): + """标点/空白差异不应阻止 L2 命中(normalize_content 应剥离)。""" + a1 = _make_article( + url="https://a/1", url_hash="aaaa", + content="The Fed raised rates today. Markets rallied strongly!" + ) + a2 = _make_article( + url="https://b/2", url_hash="bbbb", + content="The Fed, raised... rates today!!! Markets -- rallied -- strongly." + ) + assert content_hash(a1.content) == content_hash(a2.content) + with Deduper(db_path=tmp_db) as d: + d.ingest(a1) + result = d.ingest(a2) + assert result.matched_layer == DedupLayer.CONTENT + + def test_layer3_simhash_minor_rewrite(self, tmp_db): + """长文本 + 转载前后缀,落入 SimHash 层(贴近真实跨源转载场景)。""" + long_body = ( + "The Federal Reserve held interest rates steady on Wednesday, " + "citing solid economic growth and a strong labor market. " + "Markets rallied in response, with the S&P 500 gaining 1.2 percent. " + "Treasury yields fell as investors welcomed the decision. " + "Analysts expect the central bank to remain on hold through September." + ) * 2 + rewritten = "By Reuters Staff - " + long_body + " (Additional reporting by Jane Doe)" + a1 = _make_article(url="https://a/1", url_hash="aaaaa", content=long_body) + a2 = _make_article(url="https://b/2", url_hash="bbbbb", content=rewritten) + # 必要前提:content_hash 不同(否则会被 L2 截胡) + assert content_hash(a1.content) != content_hash(a2.content) + + with Deduper(db_path=tmp_db) as d: + d.ingest(a1) + result = d.ingest(a2) + assert result.is_duplicate + assert result.matched_layer == DedupLayer.SIMHASH + assert result.hamming_distance is not None + assert result.hamming_distance <= DEFAULT_HAMMING_THRESHOLD + + def test_layer3_unrelated_articles_kept(self, tmp_db): + """完全不相关文章不去重。""" + a1 = _make_article( + url="https://a/1", url_hash="aaaaa", + content="The Federal Reserve held interest rates steady on Wednesday." * 5, + ) + a2 = _make_article( + url="https://b/2", url_hash="bbbbb", + content="Apple announced a new iPhone model with revolutionary features." * 5, + title="Apple Unveils New iPhone", + ) + with Deduper(db_path=tmp_db) as d: + d.ingest(a1) + result = d.ingest(a2) + assert not result.is_duplicate + assert d.stats().total == 2 + + def test_layer3_outside_time_window_kept(self, tmp_db): + """SimHash 相近,但 publish_date 距离过远(> 30 天)不去重。""" + body = ( + "The Federal Reserve held interest rates steady on Wednesday, " + "citing solid economic growth and a strong labor market." + ) * 3 + a1 = _make_article( + url="https://a/1", url_hash="aaaa1", content=body, + publish_time="2026-01-01T09:00:00", + ) + a2 = _make_article( + url="https://b/2", url_hash="bbbb2", content=body[:50] + body, + publish_time="2026-06-16T09:00:00", + ) + assert content_hash(a1.content) != content_hash(a2.content) + with Deduper(db_path=tmp_db, time_window_days=30) as d: + d.ingest(a1) + result = d.ingest(a2) + assert not result.is_duplicate, "时间窗口外不应命中 SimHash" + + def test_threshold_zero_only_exact_simhash(self, tmp_db): + """阈值 0 → 仅当 SimHash 完全相同才视为重复(且会先被 L2 拦截)。""" + a1 = _make_article( + url="https://a/1", url_hash="aaaa1", + content="The Federal Reserve held rates steady on Wednesday." + ) + a2 = _make_article( + url="https://b/2", url_hash="bbbb2", + content="The Federal Reserve held rates unchanged on Wednesday." + ) + with Deduper(db_path=tmp_db, simhash_threshold=0) as d: + d.ingest(a1) + result = d.ingest(a2) + assert not result.is_duplicate + + def test_ingest_same_source_mixed(self, tmp_db): + """混合重复/不重复文章的同源摄入。""" + a1 = _make_article(url="https://a/1", url_hash="h1", content="Story A " * 10) + a2 = _make_article(url="https://a/2", url_hash="h2", content="Story B " * 10) + a3 = _make_article(url="https://a/3", url_hash="h3", content="Story B " * 10) # 同 a2 + a4 = _make_article(url="https://a/4", url_hash="h4", content="Story C " * 10) + + with Deduper(db_path=tmp_db) as d: + r1 = d.ingest(a1) + r2 = d.ingest(a2) + r3 = d.ingest(a3) + r4 = d.ingest(a4) + + assert not r1.is_duplicate + assert not r2.is_duplicate + assert r3.is_duplicate # L2 命中 + assert not r4.is_duplicate + assert d.stats().total == 3 + + +# --------------------------------------------------------------------------- # +# Deduper - check 不写入 +# --------------------------------------------------------------------------- # + + +class TestDeduperCheck: + """Deduper.check() 只读测试。""" + + def test_check_does_not_write(self, tmp_db): + art = _make_article() + with Deduper(db_path=tmp_db) as d: + result = d.check(art) + assert not result.is_duplicate + assert d.stats().total == 0 # check 不应入库 + + def test_check_detects_duplicate_after_ingest(self, tmp_db): + a1 = _make_article() + a2 = _make_article() + with Deduper(db_path=tmp_db) as d: + d.ingest(a1) + result = d.check(a2) + assert result.is_duplicate + assert result.matched_layer == DedupLayer.URL + + +# --------------------------------------------------------------------------- # +# Deduper - stats +# --------------------------------------------------------------------------- # + + +class TestDeduperStats: + """统计信息测试。""" + + def test_stats_aggregates_by_source(self, tmp_db): + with Deduper(db_path=tmp_db) as d: + d.ingest(_make_article( + source_id="reuters", url="https://a/1", url_hash="r000000000000001", + )) + d.ingest(_make_article( + source_id="reuters", url="https://a/2", url_hash="r000000000000002", + content="Another completely different article about markets." * 10, + )) + d.ingest(_make_article( + source_id="cnbc", url="https://b/1", url_hash="c000000000000001", + content="CNBC exclusive report on technology stocks." * 10, + )) + stats = d.stats() + assert stats.total == 3 + assert stats.by_source == {"reuters": 2, "cnbc": 1} + assert stats.earliest is not None + + +# --------------------------------------------------------------------------- # +# DedupResult 测试 +# --------------------------------------------------------------------------- # + + +class TestDedupResult: + """DedupResult 模型测试。""" + + def test_short_summary_unique(self): + r = DedupResult(url_hash="abc", is_duplicate=False) + assert r.short_summary() == "[UNIQUE] abc" + + def test_short_summary_duplicate_url(self): + r = DedupResult( + url_hash="abc", + is_duplicate=True, + matched_layer=DedupLayer.URL, + matched_url_hash="xyz", + ) + assert "[DUP/url]" in r.short_summary() + + def test_short_summary_duplicate_simhash(self): + r = DedupResult( + url_hash="abc", + is_duplicate=True, + matched_layer=DedupLayer.SIMHASH, + matched_url_hash="xyz", + hamming_distance=2, + ) + summary = r.short_summary() + assert "[DUP/simhash]" in summary + assert "hd=2" in summary diff --git a/tests/test_embedding.py b/tests/test_embedding.py new file mode 100644 index 0000000..89da9af --- /dev/null +++ b/tests/test_embedding.py @@ -0,0 +1,302 @@ +"""M5 向量生成模块单元测试。""" + +from unittest.mock import MagicMock + +import pytest +from openai import OpenAI + +from embedding.client import ( + EmbeddingConfig, + _chunked, + embed_batch, + load_embedding_config, +) +from embedding.embedder import compose_text, embed_article +from embedding.models import EmbeddingError, EmbeddingResult +from llm.models import EnTranslatedArticle, EventExtraction, Sentiment + +# --------------------------------------------------------------------------- # +# 辅助工厂 +# --------------------------------------------------------------------------- # + + +def _make_article( + *, + url_hash: str = "abc123", + source_id: str = "reuters", + source_name: str = "Reuters", + url: str = "https://example.com/1", + title: str = "Fed Holds Rates Steady", + title_zh: str = "美联储维持利率不变", + content_en: str = "The Fed held rates steady.", + content_zh: str = "美联储维持利率不变,理由是经济增长稳健。市场应声上涨。", + events: list[EventExtraction] | None = None, +) -> EnTranslatedArticle: + if events is None: + events = [ + EventExtraction( + event_type="央行决议", + stock_codes=[], + sentiment=Sentiment.POSITIVE, + importance=5, + summary_zh="美联储维持利率不变,市场反弹", + ) + ] + return EnTranslatedArticle( + source_id=source_id, + source_name=source_name, + url=url, + url_hash=url_hash, + title=title, + title_zh=title_zh, + content_en=content_en, + content_zh=content_zh, + events=events, + provider="deepseek", + model="deepseek-v4-flash", + ) + + +# --------------------------------------------------------------------------- # +# EmbeddingConfig +# --------------------------------------------------------------------------- # + + +class TestEmbeddingConfig: + """EmbeddingConfig 测试。""" + + def test_valid_config(self): + cfg = EmbeddingConfig( + model="text-embedding-v3", + api_key="sk-test", + ) + assert cfg.provider == "dashscope" + assert cfg.dimension == 1024 + + def test_missing_api_key_raises(self): + with pytest.raises(EmbeddingError, match="API_KEY"): + EmbeddingConfig(api_key="") + + +class TestLoadEmbeddingConfig: + """load_embedding_config 测试。""" + + def test_from_env(self, monkeypatch): + monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-dashscope-test") + cfg = load_embedding_config() + assert cfg.provider == "dashscope" + assert cfg.api_key == "sk-dashscope-test" + + def test_fallback_to_qwen_key(self, monkeypatch): + monkeypatch.delenv("DASHSCOPE_API_KEY", raising=False) + monkeypatch.setenv("QWEN_API_KEY", "sk-qwen-key") + cfg = load_embedding_config() + assert cfg.api_key == "sk-qwen-key" + + def test_missing_key_raises(self, monkeypatch): + monkeypatch.delenv("DASHSCOPE_API_KEY", raising=False) + monkeypatch.delenv("QWEN_API_KEY", raising=False) + with pytest.raises(EmbeddingError, match="API_KEY"): + load_embedding_config() + + +# --------------------------------------------------------------------------- # +# _chunked +# --------------------------------------------------------------------------- # + + +class TestChunked: + """_chunked 分块工具测试。""" + + def test_empty(self): + assert _chunked([], 10) == [] + + def test_single_chunk(self): + assert _chunked(["a", "b", "c"], 10) == [["a", "b", "c"]] + + def test_multiple_chunks(self): + assert _chunked(["a", "b", "c", "d", "e"], 2) == [ + ["a", "b"], ["c", "d"], ["e"] + ] + + def test_exact_fit(self): + assert _chunked(["a", "b", "c", "d"], 2) == [ + ["a", "b"], ["c", "d"] + ] + + +# --------------------------------------------------------------------------- # +# embed_batch(mock) +# --------------------------------------------------------------------------- # + + +class TestEmbedBatch: + """embed_batch 测试(mock DashScope API)。""" + + def test_empty_list(self): + cfg = EmbeddingConfig(api_key="sk-test") + client = MagicMock(spec=OpenAI) + result = embed_batch(client, cfg, []) + assert result == [] + + def test_single_text(self): + cfg = EmbeddingConfig(api_key="sk-test", batch_size=5) + client = MagicMock(spec=OpenAI) + + # Mock 返回 + mock_resp = MagicMock() + mock_resp.data = [MagicMock(embedding=[0.1] * 1024)] + client.embeddings.create.return_value = mock_resp + + result = embed_batch(client, cfg, ["测试文本"]) + assert len(result) == 1 + assert len(result[0]) == 1024 + + def test_batch_split(self): + """超过 batch_size 自动分多批。""" + cfg = EmbeddingConfig(api_key="sk-test", batch_size=3) + client = MagicMock(spec=OpenAI) + + def _make_mock(**kwargs): + input_texts = kwargs.get("input", []) + resp = MagicMock() + resp.data = [MagicMock(embedding=[0.5] * 1024) for _ in input_texts] + return resp + + client.embeddings.create.side_effect = _make_mock + + texts = ["a", "b", "c", "d", "e"] # 需要分 3+2 两批 + result = embed_batch(client, cfg, texts) + + assert len(result) == 5 + assert client.embeddings.create.call_count == 2 + + +# --------------------------------------------------------------------------- # +# compose_text +# --------------------------------------------------------------------------- # + + +class TestComposeText: + """compose_text 测试。""" + + def test_basic_composition(self): + article = _make_article() + text = compose_text(article) + assert "标题:" in text + assert "美联储维持利率不变" in text + assert "事件:" in text + assert "央行决议" in text + assert "正文:" in text + + def test_no_events(self): + article = _make_article(events=[]) + text = compose_text(article) + assert "事件:" not in text + assert "标题:" in text + assert "正文:" in text + + def test_with_stock_codes(self): + article = _make_article(events=[ + EventExtraction( + event_type="财报披露", + stock_codes=["AAPL", "TSLA"], + sentiment=Sentiment.POSITIVE, + importance=4, + summary_zh="苹果财报超预期", + ) + ]) + text = compose_text(article) + assert "AAPL" in text + assert "TSLA" in text + assert "positive" in text + + def test_truncation(self): + """超长文本应被截断。""" + long_content = "这是测试正文。" * 500 # ~2500 chars + article = _make_article(content_zh=long_content) + text = compose_text(article, max_chars=500) + assert len(text) <= 500 + + def test_empty_article(self): + article = _make_article(title_zh="", content_zh="", events=[]) + text = compose_text(article) + # 不会崩溃即可 + assert isinstance(text, str) + + +# --------------------------------------------------------------------------- # +# embed_article(mock) +# --------------------------------------------------------------------------- # + + +class TestEmbedArticle: + """embed_article 测试(mock)。""" + + def test_successful_embed(self): + cfg = EmbeddingConfig(api_key="sk-test") + client = MagicMock(spec=OpenAI) + mock_resp = MagicMock() + mock_resp.data = [MagicMock(embedding=[0.1] * 1024)] + client.embeddings.create.return_value = mock_resp + + article = _make_article() + result = embed_article(client, cfg, article) + + assert isinstance(result, EmbeddingResult) + assert result.url_hash == article.url_hash + assert result.dimension == 1024 + assert len(result.vector) == 1024 + assert result.provider == "dashscope" + assert len(result.embedded_text) > 0 + + +# --------------------------------------------------------------------------- # +# EmbeddingResult +# --------------------------------------------------------------------------- # + + +class TestEmbeddingResult: + """EmbeddingResult 模型测试。""" + + def test_minimal(self): + r = EmbeddingResult( + url_hash="abc", + source_id="reuters", + vector=[0.5] * 1024, + model="text-embedding-v3", + ) + assert r.dimension == 1024 + assert len(r.vector) == 1024 + + def test_serialization(self): + r = EmbeddingResult( + url_hash="abc", + source_id="reuters", + vector=[0.1, 0.2, 0.3], + dimension=3, + embedded_text="测试", + model="text-embedding-v3", + ) + data = r.model_dump_json() + assert "url_hash" in data + assert "vector" in data + + +# --------------------------------------------------------------------------- # +# EmbeddingError +# --------------------------------------------------------------------------- # + + +class TestEmbeddingError: + """EmbeddingError 异常测试。""" + + def test_basic(self): + err = EmbeddingError("测试错误", attempts=3) + assert err.reason == "测试错误" + assert err.attempts == 3 + assert str(err) == "测试错误" + + def test_default_attempts(self): + err = EmbeddingError("错误") + assert err.attempts == 0 diff --git a/tests/test_extractor.py b/tests/test_extractor.py new file mode 100644 index 0000000..47884f7 --- /dev/null +++ b/tests/test_extractor.py @@ -0,0 +1,143 @@ +"""M2 正文提取模块测试""" + +from pathlib import Path + +from extractor.extractor import ( + MIN_CONTENT_WORDS, + _clean_markdown, + _count_words, + _extract_author, + _extract_title, + extract_article, +) + +# ════════════════════════════════════════════════ +# 辅助函数 +# ════════════════════════════════════════════════ + +def test_count_words(): + assert _count_words("hello world") == 2 + assert _count_words("") == 0 + assert _count_words(None) == 0 + + +def test_extract_title(): + html = "Breaking News: Markets Rally" + assert "Breaking News" in _extract_title(html) + + assert _extract_title("") == "" + + +def test_extract_author(): + html = '' + assert _extract_author(html) == "John Doe" + + assert _extract_author("") == "" + + +def test_clean_markdown(): + md = """ADVERTISEMENT - Continue Reading Below +[Sign In](https://example.com/signin) +# Real Article Title +This is the actual content of the article. +It has multiple paragraphs.""" + + cleaned = _clean_markdown(md) + assert "ADVERTISEMENT" not in cleaned + assert "Sign In" not in cleaned + assert "Real Article Title" in cleaned + assert "actual content" in cleaned + + +def test_clean_markdown_preserves_content(): + md = "# Market Update\n\nStocks rose today.\n\n[Read More](https://example.com)" + cleaned = _clean_markdown(md) + assert "Market Update" in cleaned + assert "Stocks rose" in cleaned + + +# ════════════════════════════════════════════════ +# 提取引擎 +# ════════════════════════════════════════════════ + +def test_extract_article_trafilatura(tmp_path: Path): + """用 trafilatura 从 HTML 提取正文""" + html = tmp_path / "test.html" + text = ( + "Fed Raises Rates" + '' + "
" + "

The Federal Reserve raised interest rates by 25 basis points " + "today in a widely expected move. Chair Powell noted that inflation " + "remains above target but is trending downward.

" + "

Markets reacted positively, with the S&P 500 gaining 1.2%.

" + "
" + ) + html.write_text(text) + + result = extract_article( + source_id="test", + source_name="Test", + url="https://example.com/article", + url_hash="abc123", + html_path=str(html), + md_path="", + ) + + assert result.status == "success" + assert result.extractor == "trafilatura" + assert "Federal Reserve" in result.content + assert "Jane Smith" in result.author + assert result.word_count >= 30 + + +def test_extract_article_no_content(tmp_path: Path): + """无有效正文时降级""" + html = tmp_path / "empty.html" + html.write_text("Short.") + + result = extract_article( + source_id="test", source_name="Test", + url="https://example.com/nocontent", + url_hash="def456", + html_path=str(html), md_path="", + ) + + assert result.status == "no_content" + + +def test_extract_article_md_fallback(tmp_path: Path): + """HTML 不可用但 MD 可用时回退""" + md = tmp_path / "test.md" + md_text = ( + "# Market Analysis\n\n" + "This is a detailed analysis of market conditions today. " + "The dow jones industrial average showed significant movement " + "as investors reacted to economic data. " + "Trading volume was above average across major exchanges. " + "Analysts noted that technical indicators suggested continued " + "upward momentum in the near term. " + "Several key sectors led the rally including technology " + "financials and healthcare stocks. " + "The bond market also saw increased activity as yields moved lower." + ) + md.write_text(md_text) + + result = extract_article( + source_id="test", source_name="Test", + url="https://example.com/mdonly", + url_hash="ghi789", + html_path="/nonexistent/file.html", + md_path=str(md), + ) + + assert result.status == "success" + assert result.extractor == "crawl4ai_md" + assert result.word_count >= 30 + assert "Market Analysis" in result.content + + +def test_min_content_threshold(): + """MIN_CONTENT_WORDS 阈值合理""" + assert MIN_CONTENT_WORDS >= 30 + assert MIN_CONTENT_WORDS <= 100 diff --git a/tests/test_llm.py b/tests/test_llm.py new file mode 100644 index 0000000..408cad0 --- /dev/null +++ b/tests/test_llm.py @@ -0,0 +1,622 @@ +"""M4 LLM 翻译 + 事件抽取模块单元测试。""" + +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from openai import OpenAI + +from extractor.models import ProcessedArticle +from llm.client import LLMConfig, load_llm_config +from llm.extractor import ( + MAX_CONTENT_CHARS, + PromptTemplate, + _extract_json_object, + parse_translation_json, + translate_and_extract, +) +from llm.models import ( + INTERNATIONAL_EVENT_TYPES, + EnTranslatedArticle, + EventExtraction, + LLMCallError, + LLMTranslationOutput, + Sentiment, +) + +# --------------------------------------------------------------------------- # +# 辅助工厂 +# --------------------------------------------------------------------------- # + + +def _make_article( + *, + url: str = "https://www.reuters.com/business/1", + url_hash: str = "abc1234567890000", + source_id: str = "reuters", + source_name: str = "Reuters", + title: str = "Fed Holds Rates Steady as Markets Rally", + content: str = ( + "The Federal Reserve held interest rates steady on Wednesday, " + "citing solid economic growth and a strong labor market. " + "Markets rallied in response, with the S&P 500 gaining 1.2 percent." + ), + publish_time: str = "2026-06-16T10:00:00", + word_count: int = 0, +) -> ProcessedArticle: + return ProcessedArticle( + source_id=source_id, + source_name=source_name, + url=url, + url_hash=url_hash, + title=title, + content=content, + publish_time=publish_time, + word_count=word_count or len(content.split()), + ) + + +# --------------------------------------------------------------------------- # +# Sentiment / 枚举 +# --------------------------------------------------------------------------- # + + +class TestSentiment: + """Sentiment 枚举测试。""" + + def test_values(self): + assert Sentiment.POSITIVE == "positive" + assert Sentiment.NEUTRAL == "neutral" + assert Sentiment.NEGATIVE == "negative" + + def test_from_string(self): + assert Sentiment("positive") == Sentiment.POSITIVE + assert Sentiment("neutral") == Sentiment.NEUTRAL + assert Sentiment("negative") == Sentiment.NEGATIVE + + +# --------------------------------------------------------------------------- # +# EventExtraction 模型校验 +# --------------------------------------------------------------------------- # + + +class TestEventExtraction: + """EventExtraction 模型测试。""" + + def test_valid_event(self): + ev = EventExtraction( + event_type="财报披露", + stock_codes=["AAPL", "TSLA"], + sentiment="positive", + importance=4, + summary_zh="苹果第三季度营收超预期", + ) + assert ev.event_type == "财报披露" + assert ev.stock_codes == ["AAPL", "TSLA"] + assert ev.sentiment == Sentiment.POSITIVE + assert ev.importance == 4 + + def test_stock_codes_filtered_and_uppercased(self): + ev = EventExtraction( + event_type="并购收购", + stock_codes=["aapl", " msft ", "", "INVALID123", "GOOGL"], + sentiment="neutral", + importance=3, + summary_zh="微软收购测试", + ) + # INVALID123 > 5 chars → 过滤;空 → 过滤;小写 → 大写;去重 + assert ev.stock_codes == ["AAPL", "MSFT", "GOOGL"] + + def test_empty_stock_codes(self): + ev = EventExtraction( + event_type="宏观经济", + stock_codes=[], + sentiment="neutral", + importance=2, + summary_zh="GDP 数据发布", + ) + assert ev.stock_codes == [] + + def test_importance_bounds(self): + # 1 和 5 都能通过 + ev1 = EventExtraction( + event_type="宏观经济", sentiment="neutral", importance=1, summary_zh="t1" + ) + assert ev1.importance == 1 + ev5 = EventExtraction( + event_type="央行决议", sentiment="negative", importance=5, summary_zh="t5" + ) + assert ev5.importance == 5 + + def test_importance_out_of_range_rejected(self): + with pytest.raises(Exception): + EventExtraction( + event_type="其他", sentiment="neutral", importance=0, summary_zh="t" + ) + with pytest.raises(Exception): + EventExtraction( + event_type="其他", sentiment="neutral", importance=6, summary_zh="t" + ) + + def test_invalid_sentiment_rejected(self): + with pytest.raises(Exception): + EventExtraction( + event_type="其他", stock_codes=[], sentiment="happy", importance=3, + summary_zh="t", + ) + + def test_event_type_normalized(self): + """空 event_type 默认"其他"。""" + ev = EventExtraction( + event_type="", sentiment="neutral", importance=2, summary_zh="测试" + ) + assert ev.event_type == "其他" + + def test_summary_zh_max_length_rejected(self): + """超长 summary_zh(>200 字符)直接拒绝。""" + long_summary = "测试" * 150 # 300 chars > 200 + with pytest.raises(Exception): + EventExtraction( + event_type="行业动态", + sentiment="neutral", + importance=2, + summary_zh=long_summary, + ) + + +# --------------------------------------------------------------------------- # +# LLMTranslationOutput +# --------------------------------------------------------------------------- # + + +class TestLLMTranslationOutput: + """LLMTranslationOutput 模型测试。""" + + def test_valid_full_output(self): + data = { + "title_zh": "美联储维持利率不变,市场上涨", + "content_zh": "美联储周三维持利率不变...", + "events": [ + { + "event_type": "央行决议", + "stock_codes": [], + "sentiment": "positive", + "importance": 5, + "summary_zh": "美联储维持利率不变", + } + ], + } + out = LLMTranslationOutput.model_validate(data) + assert out.title_zh == data["title_zh"] + assert len(out.events) == 1 + assert out.events[0].event_type == "央行决议" + + def test_no_events(self): + data = { + "title_zh": "每日市场简报", + "content_zh": "今日市场整体平淡...", + "events": [], + } + out = LLMTranslationOutput.model_validate(data) + assert out.events == [] + + def test_missing_title_zh_rejected(self): + data = { + "content_zh": "正文...", + "events": [], + } + with pytest.raises(Exception): + LLMTranslationOutput.model_validate(data) + + def test_missing_content_zh_rejected(self): + data = { + "title_zh": "标题", + "events": [], + } + with pytest.raises(Exception): + LLMTranslationOutput.model_validate(data) + + +# --------------------------------------------------------------------------- # +# EnTranslatedArticle +# --------------------------------------------------------------------------- # + + +class TestEnTranslatedArticle: + """EnTranslatedArticle 模型测试。""" + + def test_minimal_construction(self): + article = EnTranslatedArticle( + source_id="reuters", + source_name="Reuters", + url="https://example.com/1", + url_hash="abc123", + title="Fed Holds Rates", + title_zh="美联储维持利率", + content_en="The Fed held rates steady.", + content_zh="美联储维持利率不变。", + provider="deepseek", + model="deepseek-v4-flash", + ) + assert article.events == [] + assert article.word_count_zh == 0 + + def test_with_events(self): + article = EnTranslatedArticle( + source_id="reuters", + source_name="Reuters", + url="https://example.com/1", + url_hash="abc123", + title="Apple Earnings", + title_zh="苹果财报", + content_en="Apple reported record earnings.", + content_zh="苹果公布了创纪录的财报。", + events=[ + EventExtraction( + event_type="财报披露", + stock_codes=["AAPL"], + sentiment="positive", + importance=4, + summary_zh="苹果财报超预期", + ) + ], + provider="deepseek", + model="deepseek-v4-flash", + ) + assert len(article.events) == 1 + assert "AAPL" in article.short_summary() + + def test_short_summary_no_events(self): + article = EnTranslatedArticle( + source_id="reuters", + source_name="Reuters", + url="https://example.com/1", + url_hash="abc123", + title="Market Wrap", + title_zh="市场综述", + content_en="Markets were flat today.", + content_zh="今日市场持平。", + provider="deepseek", + model="deepseek-v4-flash", + ) + assert "-" in article.short_summary() or "0events" in article.short_summary() + + +# --------------------------------------------------------------------------- # +# PromptTemplate +# --------------------------------------------------------------------------- # + + +class TestPromptTemplate: + """PromptTemplate 测试。""" + + def test_parse_and_render(self, tmp_path: Path): + """测试模板解析和渲染。""" + prompt_content = """# 测试标题 + +## System Prompt + +你是翻译助手。 + +--- + +## User Input + +标题: {title} +来源: {source_name} +正文: {content} +""" + prompt_path = tmp_path / "test_prompt.md" + prompt_path.write_text(prompt_content, encoding="utf-8") + + tpl = PromptTemplate(template_path=prompt_path) + article = _make_article() + system, user = tpl.render(article) + + assert "翻译助手" in system + assert article.title in user + assert article.source_name in user + assert article.content in user + + def test_content_truncation(self, tmp_path: Path): + """测试超长正文截断。""" + prompt_content = """## System Prompt +你是助手。 +--- + +## User Input +正文: {content} +""" + prompt_path = tmp_path / "test_prompt.md" + prompt_path.write_text(prompt_content, encoding="utf-8") + + tpl = PromptTemplate(template_path=prompt_path) + long_content = "X" * (MAX_CONTENT_CHARS + 500) + article = _make_article(content=long_content) + + _system, user = tpl.render(article) + assert "正文过长已截断" in user + assert len("X" * MAX_CONTENT_CHARS) + len("\n\n[正文过长已截断]") < len(user) + + +# --------------------------------------------------------------------------- # +# JSON 提取 +# --------------------------------------------------------------------------- # + + +class TestExtractJsonObject: + """_extract_json_object 函数测试。""" + + def test_plain_json(self): + raw = '{"key": "value"}' + assert _extract_json_object(raw) == '{"key": "value"}' + + def test_json_with_fence(self): + raw = '```json\n{"key": "value"}\n```' + assert _extract_json_object(raw) == '{"key": "value"}' + + def test_json_with_text_before(self): + raw = 'Here is the result:\n{"key": "value"}' + assert _extract_json_object(raw) == '{"key": "value"}' + + def test_nested_braces(self): + raw = '{"outer": {"inner": [1, 2, 3]}}' + assert _extract_json_object(raw) == raw.strip() + + def test_empty_string(self): + assert _extract_json_object("") == "" + + +# --------------------------------------------------------------------------- # +# parse_translation_json +# --------------------------------------------------------------------------- # + + +class TestParseTranslationJson: + """parse_translation_json 函数测试。""" + + def test_valid_json(self): + raw = json.dumps({ + "title_zh": "测试标题", + "content_zh": "测试正文", + "events": [], + }) + result = parse_translation_json(raw) + assert result.title_zh == "测试标题" + assert result.content_zh == "测试正文" + + def test_invalid_json(self): + with pytest.raises(LLMCallError, match="JSON 解析失败"): + parse_translation_json("not valid json {{{") + + def test_non_object(self): + with pytest.raises(LLMCallError, match="非对象"): + parse_translation_json("[1, 2, 3]") + + def test_schema_validation_fails(self): + """缺少必填字段时抛出 LLMCallError。""" + raw = json.dumps({"title_zh": "标题"}) # 缺少 content_zh + with pytest.raises(LLMCallError, match="schema 校验失败"): + parse_translation_json(raw) + + +# --------------------------------------------------------------------------- # +# LLMConfig / load_llm_config +# --------------------------------------------------------------------------- # + + +class TestLLMConfig: + """LLMConfig 测试。""" + + def test_valid_config(self): + cfg = LLMConfig( + provider="deepseek", + model="deepseek-chat", + api_key="sk-test", + base_url="https://api.deepseek.com", + ) + assert cfg.provider == "deepseek" + + def test_empty_api_key_raises(self): + with pytest.raises(ValueError, match="API key 为空"): + LLMConfig( + provider="deepseek", + model="deepseek-chat", + api_key="", + base_url="https://api.deepseek.com", + ) + + +class TestLoadLLMConfig: + """load_llm_config 函数测试。""" + + def test_deepseek_from_env(self, monkeypatch): + """从环境变量构造 DeepSeek 配置。""" + monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-deepseek-test-key") + config = load_llm_config(provider="deepseek") + assert config.provider == "deepseek" + assert config.api_key == "sk-deepseek-test-key" + assert "deepseek" in config.base_url + + def test_qwen_fallback_to_dashscope_key(self, monkeypatch): + """Qwen 的 API Key 可回退到 DASHSCOPE_API_KEY。""" + monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-dashscope-key") + monkeypatch.delenv("QWEN_API_KEY", raising=False) + config = load_llm_config(provider="qwen") + assert config.provider == "qwen" + assert config.api_key == "sk-dashscope-key" + + def test_unknown_provider_raises(self): + with pytest.raises(ValueError, match="未知 LLM provider"): + load_llm_config(provider="openai") + + def test_missing_api_key_raises(self, monkeypatch): + """未配置 API Key 时应抛出明确错误。""" + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + with pytest.raises(ValueError, match="API key 未配置"): + load_llm_config(provider="deepseek") + + +# --------------------------------------------------------------------------- # +# translate_and_extract(mock LLM) +# --------------------------------------------------------------------------- # + + +class TestTranslateAndExtract: + """translate_and_extract 测试(mock LLM 响应)。""" + + def test_successful_translation(self, monkeypatch): + """Mock LLM 返回有效 JSON。""" + config = LLMConfig( + provider="deepseek", + model="test-model", + api_key="sk-test", + base_url="https://test.api", + ) + article = _make_article() + + # Mock OpenAI client + mock_client = MagicMock(spec=OpenAI) + mock_response = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[0].message.content = json.dumps({ + "title_zh": "美联储维持利率不变", + "content_zh": "美联储周三维持利率不变,理由是经济增长稳健。", + "events": [ + { + "event_type": "央行决议", + "stock_codes": [], + "sentiment": "positive", + "importance": 5, + "summary_zh": "美联储维持利率不变,市场反弹", + } + ], + }) + mock_response.usage = MagicMock() + mock_response.usage.prompt_tokens = 500 + mock_response.usage.completion_tokens = 200 + mock_client.chat.completions.create.return_value = mock_response + + result = translate_and_extract( + client=mock_client, + config=config, + article=article, + ) + + assert result.title_zh == "美联储维持利率不变" + assert len(result.events) == 1 + assert result.events[0].event_type == "央行决议" + assert result.provider == "deepseek" + assert result.prompt_tokens == 500 + assert result.completion_tokens == 200 + assert result.word_count_zh > 0 + + def test_empty_content_zh_retries(self, monkeypatch): + """LLM 返回空 content_zh 时触发重试。""" + config = LLMConfig( + provider="deepseek", + model="test-model", + api_key="sk-test", + base_url="https://test.api", + ) + article = _make_article() + + mock_client = MagicMock(spec=OpenAI) + # 始终返回空 content_zh + mock_response = MagicMock() + mock_response.choices = [MagicMock()] + mock_response.choices[0].message.content = json.dumps({ + "title_zh": "标题", + "content_zh": "", + "events": [], + }) + mock_response.usage = MagicMock() + mock_response.usage.prompt_tokens = 100 + mock_response.usage.completion_tokens = 10 + mock_client.chat.completions.create.return_value = mock_response + + with pytest.raises(LLMCallError, match="放弃"): + translate_and_extract( + client=mock_client, + config=config, + article=article, + max_attempts=2, # 减少重试加速测试 + ) + + def test_retry_on_json_parse_failure(self, monkeypatch): + """前 N-1 次返回无效 JSON,最后一次成功。""" + config = LLMConfig( + provider="deepseek", + model="test-model", + api_key="sk-test", + base_url="https://test.api", + ) + article = _make_article() + + mock_client = MagicMock(spec=OpenAI) + # 第一次失败,第二次成功 + mock_client.chat.completions.create.side_effect = [ + _mock_chat_response("not valid {{{ json"), + _mock_chat_response(json.dumps({ + "title_zh": "测试标题", + "content_zh": "测试正文", + "events": [], + })), + ] + + result = translate_and_extract( + client=mock_client, + config=config, + article=article, + max_attempts=3, + ) + + assert result.attempts == 2 # 第二次成功 + assert result.title_zh == "测试标题" + + +def _mock_chat_response(content: str) -> MagicMock: + """Helper:构造 mock OpenAI chat completion 响应。""" + resp = MagicMock() + resp.choices = [MagicMock()] + resp.choices[0].message.content = content + resp.usage = MagicMock() + resp.usage.prompt_tokens = 100 + resp.usage.completion_tokens = 50 + return resp + + +# --------------------------------------------------------------------------- # +# INTERNATIONAL_EVENT_TYPES +# --------------------------------------------------------------------------- # + + +class TestEventTypes: + """事件类型常量测试。""" + + def test_has_expected_types(self): + assert "财报披露" in INTERNATIONAL_EVENT_TYPES + assert "并购收购" in INTERNATIONAL_EVENT_TYPES + assert "央行决议" in INTERNATIONAL_EVENT_TYPES + assert "地缘政治" in INTERNATIONAL_EVENT_TYPES + assert len(INTERNATIONAL_EVENT_TYPES) >= 10 + + +# --------------------------------------------------------------------------- # +# LLMCallError +# --------------------------------------------------------------------------- # + + +class TestLLMCallError: + """LLMCallError 异常测试。""" + + def test_basic(self): + err = LLMCallError("测试错误", attempts=3) + assert err.reason == "测试错误" + assert err.attempts == 3 + assert str(err) == "测试错误" + + def test_default_attempts(self): + err = LLMCallError("错误") + assert err.attempts == 0 diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py new file mode 100644 index 0000000..769df86 --- /dev/null +++ b/tests/test_mcp_server.py @@ -0,0 +1,188 @@ +"""M8 MCP 服务模块单元测试。""" + +from unittest.mock import MagicMock, patch + +from vectorstore.models import SearchFilter + +# --------------------------------------------------------------------------- # +# _fmt_results +# --------------------------------------------------------------------------- # + + +class TestFmtResults: + """_fmt_results 格式化测试。""" + + def test_empty_hits(self): + from mcp_server.server import _fmt_results + result = _fmt_results([], "test query") + assert "未找到" in result + + def test_with_results(self): + from mcp_server.server import _fmt_results + hits = [{ + "title": "Fed Holds Rates", + "title_zh": "美联储维持利率", + "source": "reuters", + "score": 0.95, + "url": "https://example.com/1", + "events": [{ + "event_type": "央行决议", + "sentiment": "neutral", + "importance": 5, + "stock_codes": [], + "summary_zh": "美联储维持利率不变", + }], + }] + result = _fmt_results(hits, "Fed") + assert "美联储维持利率" in result + assert "reuters" in result + assert "0.95" in result + + def test_with_stock_codes(self): + from mcp_server.server import _fmt_results + hits = [{ + "title": "Apple Earnings", + "title_zh": "苹果财报", + "source": "reuters", + "score": 0.88, + "url": "https://example.com/1", + "events": [{ + "event_type": "财报披露", + "sentiment": "positive", + "importance": 4, + "stock_codes": ["AAPL"], + "summary_zh": "苹果财报超预期", + }], + }] + result = _fmt_results(hits, "AAPL") + assert "AAPL" in result + assert "🟢利好" in result + + +# --------------------------------------------------------------------------- # +# _load_today_events +# --------------------------------------------------------------------------- # + + +class TestLoadTodayEvents: + """_load_today_events 测试。""" + + def test_no_data_dir(self): + from mcp_server.server import _load_today_events + events = _load_today_events("20990101") + assert events == [] + + @patch("mcp_server.server.Path.is_dir") + @patch("mcp_server.server.Path.glob") + def test_loads_high_importance_only(self, mock_glob, mock_is_dir): + import json + + from mcp_server.server import _load_today_events + + mock_is_dir.return_value = True + # 创建 mock 文件 + mock_file = MagicMock() + mock_file.name = "test.json" + mock_file.read_text.return_value = json.dumps({ + "title": "Test", + "title_zh": "测试", + "url": "https://x.com/1", + "source_id": "reuters", + "events": [ + {"importance": 5, "event_type": "央行决议", "sentiment": "neutral", + "stock_codes": [], "summary_zh": "t1"}, + {"importance": 2, "event_type": "其他", "sentiment": "neutral", + "stock_codes": [], "summary_zh": "t2"}, + ], + }) + mock_glob.return_value = [mock_file] + + events = _load_today_events("20260621") + # 只有 importance ≥ 4 的 + assert len(events) == 1 + assert events[0]["importance"] == 5 + + +# --------------------------------------------------------------------------- # +# SearchFilter +# --------------------------------------------------------------------------- # + + +class TestSearchFilterMCP: + """SearchFilter 用于 MCP 的测试。""" + + def test_stock_filter(self): + f = SearchFilter(stock_codes=["AAPL"]) + assert "AAPL" in f.stock_codes + + def test_sentiment_filter(self): + f = SearchFilter(sentiment="positive") + assert f.sentiment == "positive" + + +# --------------------------------------------------------------------------- # +# _search(mock) +# --------------------------------------------------------------------------- # + + +class TestSearch: + """_search 函数测试(mock backend)。""" + + @patch("mcp_server.server._get_backend") + def test_search_returns_formatted(self, mock_backend): + from mcp_server.server import _search + + # Mock backend + be = MagicMock() + mock_backend.return_value = be + + # Mock embed_batch + with patch("mcp_server.server.embed_batch") as mock_embed: + mock_embed.return_value = [[0.1] * 1024] + + # Mock vector_store.query + mock_result = MagicMock() + mock_result.title = "Test" + mock_result.title_zh = "测试" + mock_result.url = "https://x.com/1" + mock_result.source_id = "reuters" + mock_result.score = 0.9 + mock_result.publish_time = "2026-06-21" + mock_result.events = [] + mock_result.content_zh_preview = "" + be.vector_store.query.return_value = [mock_result] + + hits = _search("test") + assert len(hits) == 1 + assert hits[0]["title"] == "Test" + + +# --------------------------------------------------------------------------- # +# MCP 模块导入 +# --------------------------------------------------------------------------- # + + +class TestMCPImport: + """MCP 服务模块导入测试。""" + + def test_mcp_object_imports(self): + """确保 mcp FastMCP 对象可导入。""" + from mcp_server.server import mcp + assert mcp is not None + assert mcp.name is not None + + def test_tools_registered(self): + """确保 5 个工具函数存在且已注册。""" + from mcp_server.server import ( + get_stats, + get_today_events, + search_by_sentiment, + search_by_stock, + search_news, + ) + # 验证函数存在且可调用 + assert callable(search_news) + assert callable(search_by_stock) + assert callable(search_by_sentiment) + assert callable(get_today_events) + assert callable(get_stats) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 0000000..6b81b21 --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,122 @@ +"""M7 调度与日报模块单元测试。""" + +from unittest.mock import patch + +from scheduler.pipeline import PipelineResult, StepResult + +# --------------------------------------------------------------------------- # +# StepResult +# --------------------------------------------------------------------------- # + + +class TestStepResult: + """StepResult 测试。""" + + def test_success_step(self): + sr = StepResult(name="extract", success=True, elapsed_sec=5.0, message="17 篇") + assert sr.name == "extract" + assert sr.success + assert sr.message == "17 篇" + + def test_failed_step(self): + sr = StepResult(name="translate", success=False, elapsed_sec=30.0, + message="API timeout") + assert not sr.success + assert "timeout" in sr.message + + +# --------------------------------------------------------------------------- # +# PipelineResult +# --------------------------------------------------------------------------- # + + +class TestPipelineResult: + """PipelineResult 测试。""" + + def test_all_success(self): + result = PipelineResult(steps=[ + StepResult(name="extract", success=True, elapsed_sec=1.0), + StepResult(name="dedup", success=True, elapsed_sec=0.5), + ]) + assert result.all_success + assert result.success_count == 2 + + def test_partial_failure(self): + result = PipelineResult(steps=[ + StepResult(name="extract", success=True, elapsed_sec=1.0), + StepResult(name="translate", success=False, elapsed_sec=30.0), + StepResult(name="embed", success=True, elapsed_sec=2.0), + ]) + assert not result.all_success + assert result.success_count == 2 + + def test_empty(self): + result = PipelineResult() + assert result.all_success # 空集合 vacuously true + assert result.success_count == 0 + + +# --------------------------------------------------------------------------- # +# run_pipeline(mock 各模块) +# --------------------------------------------------------------------------- # + + +class TestRunPipeline: + """run_pipeline 测试(mock 各步骤)。""" + + @patch("scheduler.pipeline.run_step_extract") + @patch("scheduler.pipeline.run_step_dedup") + @patch("scheduler.pipeline.run_step_embed") + @patch("scheduler.pipeline.run_step_index") + def test_pipeline_runs_all_steps(self, mock_idx, mock_emb, mock_dedup, mock_ext): + from scheduler.pipeline import run_pipeline + + mock_ext.return_value = StepResult(name="extract", success=True, elapsed_sec=1) + mock_dedup.return_value = StepResult(name="dedup", success=True, elapsed_sec=1) + mock_emb.return_value = StepResult(name="embed", success=True, elapsed_sec=1) + mock_idx.return_value = StepResult(name="index", success=True, elapsed_sec=1) + + result = run_pipeline("20260621", skip_report=True) + assert result.success_count >= 4 + + @patch("scheduler.pipeline.run_step_extract") + @patch("scheduler.pipeline.run_step_dedup") + def test_pipeline_continues_on_failure(self, mock_dedup, mock_ext): + from scheduler.pipeline import run_pipeline + + mock_ext.return_value = StepResult(name="extract", success=False, elapsed_sec=1, + message="error") + mock_dedup.return_value = StepResult(name="dedup", success=True, elapsed_sec=1) + + result = run_pipeline("20260621", steps=["extract", "dedup"], skip_report=True) + # extract 失败但 dedup 仍然执行 + assert result.success_count == 1 + + def test_unknown_step_skipped(self): + from scheduler.pipeline import run_pipeline + result = run_pipeline("20260621", steps=["nonexistent_step"], skip_report=True) + assert len(result.steps) == 1 + assert not result.steps[0].success + + +# --------------------------------------------------------------------------- # +# generate_report(无数据场景) +# --------------------------------------------------------------------------- # + + +class TestGenerateReport: + """generate_report 测试。""" + + @patch("scheduler.reporter._load_events_window") + @patch("scheduler.reporter._collect_stats_window") + def test_no_data_returns_none(self, mock_stats, mock_events): + from scheduler.reporter import generate_report + + mock_events.return_value = [] + mock_stats.return_value = { + "proc": 0, "deduped": 0, "emb_count": 0, + "qdrant_count": 0, "raw_total": 0, "raw_by_source": {}, + } + + result = generate_report() + assert result is None diff --git a/tests/test_vectorstore.py b/tests/test_vectorstore.py new file mode 100644 index 0000000..db87691 --- /dev/null +++ b/tests/test_vectorstore.py @@ -0,0 +1,297 @@ +"""M6 Qdrant 向量存储模块单元测试。""" + + + +from vectorstore.client import ( + DEFAULT_COLLECTION, + VectorStore, + make_qdrant_client, + url_hash_to_uuid, +) +from vectorstore.models import CollectionInfo, SearchFilter, SearchResult + +# --------------------------------------------------------------------------- # +# url_hash_to_uuid +# --------------------------------------------------------------------------- # + + +class TestUrlHashToUuid: + """url_hash_to_uuid 函数测试。""" + + def test_deterministic(self): + h = "abc1234567890000" + assert url_hash_to_uuid(h) == url_hash_to_uuid(h) + + def test_different_hash_different_uuid(self): + assert url_hash_to_uuid("aaa1111111111111") != url_hash_to_uuid("bbb2222222222222") + + def test_valid_uuid_format(self): + import uuid + result = url_hash_to_uuid("abc1234567890000") + uuid.UUID(result) # 应不抛异常 + + +# --------------------------------------------------------------------------- # +# make_qdrant_client +# --------------------------------------------------------------------------- # + + +class TestMakeQdrantClient: + """make_qdrant_client 测试。""" + + def test_memory_mode(self): + client = make_qdrant_client(memory=True) + assert client is not None + client.close() + + def test_custom_path(self, tmp_path): + path = str(tmp_path / "qdrant_test") + client = make_qdrant_client(path=path) + assert client is not None + client.close() + + +# --------------------------------------------------------------------------- # +# VectorStore — Collection 管理 +# --------------------------------------------------------------------------- # + + +class TestVectorStoreInit: + """VectorStore Collection 初始化测试。""" + + def test_init_collection_creates(self): + client = make_qdrant_client(memory=True) + store = VectorStore(client) + try: + store.init_collection() + info = store.info() + assert info.exists + assert info.name == DEFAULT_COLLECTION + finally: + store.close() + + def test_init_idempotent(self): + client = make_qdrant_client(memory=True) + store = VectorStore(client) + try: + store.init_collection() + store.init_collection() # 第二次不应报错 + assert store.info().exists + finally: + store.close() + + def test_recreate(self): + client = make_qdrant_client(memory=True) + store = VectorStore(client) + try: + store.init_collection() + store.init_collection(recreate=True) + assert store.info().exists + finally: + store.close() + + def test_info_nonexistent(self): + client = make_qdrant_client(memory=True) + store = VectorStore(client, collection_name="nonexistent_test") + try: + info = store.info() + assert not info.exists + finally: + store.close() + + +# --------------------------------------------------------------------------- # +# VectorStore — upsert + query +# --------------------------------------------------------------------------- # + + +class TestVectorStoreUpsert: + """VectorStore upsert 测试。""" + + def test_upsert_and_count(self): + client = make_qdrant_client(memory=True) + store = VectorStore(client) + try: + store.init_collection() + points = [ + { + "id": "hash0000000000001", + "vector": [0.1] * 1024, + "payload": { + "title": "Test Article", + "title_zh": "测试文章", + "url": "https://example.com/1", + "source_id": "reuters", + }, + }, + { + "id": "hash0000000000002", + "vector": [0.2] * 1024, + "payload": { + "title": "Another Article", + "title_zh": "另一篇文章", + "url": "https://example.com/2", + "source_id": "cnbc", + }, + }, + ] + count = store.upsert(points) + assert count == 2 + assert store.count() == 2 + finally: + store.close() + + def test_upsert_idempotent(self): + client = make_qdrant_client(memory=True) + store = VectorStore(client) + try: + store.init_collection() + points = [{ + "id": "hash0000000000001", + "vector": [0.1] * 1024, + "payload": {"title": "Original"}, + }] + store.upsert(points) + + # 同一 id 第二次写入(更新) + points2 = [{ + "id": "hash0000000000001", + "vector": [0.9] * 1024, + "payload": {"title": "Updated"}, + }] + store.upsert(points2) + + assert store.count() == 1 # 不应增加 + finally: + store.close() + + +class TestVectorStoreQuery: + """VectorStore query 测试。""" + + def test_query_returns_results(self): + client = make_qdrant_client(memory=True) + store = VectorStore(client) + try: + store.init_collection() + # 写入 3 条 + for i in range(3): + store.upsert([{ + "id": f"hash{i:016d}", + "vector": [float(i) / 10] * 1024, + "payload": { + "title": f"Article {i}", + "title_zh": f"文章 {i}", + "url": f"https://example.com/{i}", + "source_id": "reuters", + "events": [], + }, + }]) + + # 查询 + query_vec = [0.15] * 1024 # 接近 hash0 和 hash1 + results = store.query(query_vec, top_k=2) + assert len(results) == 2 + assert results[0].score > 0 # 有相似度分数 + finally: + store.close() + + def test_query_with_filter(self): + client = make_qdrant_client(memory=True) + store = VectorStore(client) + try: + store.init_collection() + for i in range(5): + store.upsert([{ + "id": f"hash{i:016d}", + "vector": [0.5] * 1024, + "payload": { + "title": f"A{i}", + "title_zh": f"文{i}", + "url": f"https://x.com/{i}", + "source_id": "reuters" if i < 3 else "cnbc", + "events": [], + }, + }]) + + # 过滤只查 reuters + sf = SearchFilter(source_id="reuters") + results = store.query([0.5] * 1024, top_k=10, search_filter=sf) + assert all(r.source_id == "reuters" for r in results) + finally: + store.close() + + +# --------------------------------------------------------------------------- # +# SearchFilter / SearchResult 模型 +# --------------------------------------------------------------------------- # + + +class TestSearchFilter: + """SearchFilter 模型测试。""" + + def test_empty_filter(self): + f = SearchFilter() + assert f.source_id is None + + def test_source_id_filter(self): + f = SearchFilter(source_id="reuters") + assert f.source_id == "reuters" + + def test_multi_condition(self): + f = SearchFilter( + source_ids=["reuters", "cnbc"], + sentiment="positive", + importance_min=3, + publish_date_from="2026-06-01", + publish_date_to="2026-06-30", + ) + assert f.sentiment == "positive" + assert f.importance_min == 3 + + +class TestSearchResult: + """SearchResult 模型测试。""" + + def test_basic(self): + r = SearchResult( + url_hash="abc", + score=0.95, + title="Fed Holds Rates", + title_zh="美联储维持利率", + url="https://example.com/1", + source_id="reuters", + ) + assert r.score == 0.95 + assert "美联储" in r.short_summary() + + def test_with_events(self): + r = SearchResult( + url_hash="abc", + score=0.88, + title="Apple Earnings", + title_zh="苹果财报", + source_id="reuters", + events=[{ + "event_type": "财报披露", + "stock_codes": ["AAPL"], + "sentiment": "positive", + "importance": 4, + "summary_zh": "苹果财报超预期", + }], + ) + assert "AAPL" in r.short_summary() + + +class TestCollectionInfo: + """CollectionInfo 模型测试。""" + + def test_basic(self): + info = CollectionInfo(name="test", exists=True, vectors_count=42) + assert info.exists + assert info.vectors_count == 42 + + def test_defaults(self): + info = CollectionInfo(name="empty", exists=False) + assert info.vectors_count == 0 + assert info.indexed_vectors_count is None diff --git a/translator/__init__.py b/translator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vectorstore/__init__.py b/vectorstore/__init__.py new file mode 100644 index 0000000..39f982b --- /dev/null +++ b/vectorstore/__init__.py @@ -0,0 +1,37 @@ +"""Qdrant 向量知识库模块 (M6)。 + +公共 API: + - VectorStore: Qdrant 封装(init / upsert / query / info / count) + - SearchFilter / SearchResult / CollectionInfo: 数据模型 + - make_qdrant_client: 工厂(内存/本地文件/远程 HTTP) + - ingest_all_embeddings / search_news / get_collection_info: 管道 +""" + +from vectorstore.client import ( + DEFAULT_COLLECTION, + DEFAULT_VECTOR_DIM, + VectorStore, + make_qdrant_client, +) +from vectorstore.models import CollectionInfo, SearchFilter, SearchResult +from vectorstore.pipeline import ( + get_collection_info, + ingest_all_embeddings, + search_news, +) + +__all__ = [ + # 客户端 + "DEFAULT_COLLECTION", + "DEFAULT_VECTOR_DIM", + "VectorStore", + "make_qdrant_client", + # 模型 + "CollectionInfo", + "SearchFilter", + "SearchResult", + # 管道 + "get_collection_info", + "ingest_all_embeddings", + "search_news", +] diff --git a/vectorstore/client.py b/vectorstore/client.py new file mode 100644 index 0000000..06241d9 --- /dev/null +++ b/vectorstore/client.py @@ -0,0 +1,330 @@ +"""Qdrant 客户端封装 (M6)。 + +核心: + - 连接: 本地文件模式(默认,无需 Docker)或 HTTP 远程模式 + - 初始化 Collection: 1024 维 / 余弦距离 + - upsert: 幂等写入(url_hash 转 UUID 做 point ID) + - query: 语义检索 + 结构化过滤 + - info / count: 运维辅助 +""" + +import logging +import os +import uuid +from pathlib import Path + +import yaml +from qdrant_client import QdrantClient +from qdrant_client.http.models import ( + DatetimeRange, + Distance, + FieldCondition, + Filter, + MatchAny, + MatchValue, + PointStruct, + Range, + VectorParams, +) + +from vectorstore.models import CollectionInfo, SearchFilter, SearchResult + +logger = logging.getLogger(__name__) + +# 默认配置 +DEFAULT_COLLECTION = "en_finance_news" +DEFAULT_VECTOR_DIM = 1024 +DEFAULT_DISTANCE = Distance.COSINE +DEFAULT_STORAGE_PATH = Path("data/qdrant_storage") + +# UUID namespace for url_hash -> UUID conversion(确定性,便于幂等 upsert) +_UUID_NAMESPACE = uuid.UUID("e1d2c3b4-a5f6-7890-abcd-ef1234567890") + + +def url_hash_to_uuid(url_hash: str) -> str: + """把 url_hash 转为 UUID 字符串(point ID 要求)。 + + 使用 uuid5 保证确定性——相同 url_hash 总是得到相同 UUID。 + """ + return str(uuid.uuid5(_UUID_NAMESPACE, url_hash)) + + +def _load_qdrant_config() -> dict: + """从 system.yaml 加载 qdrant 段配置。""" + config_path = Path("configs/system.yaml") + if config_path.exists(): + try: + with open(config_path, encoding="utf-8") as f: + raw = yaml.safe_load(f) + return raw.get("qdrant", {}) + except Exception: + pass + return {} + + +def make_qdrant_client( + *, + memory: bool = False, + path: str | None = None, +) -> QdrantClient: + """构造 QdrantClient。 + + 模式优先级: + 1. memory=True → 内存模式(测试用) + 2. path 非空 → 本地文件模式(嵌入式运行,无需 Docker) + 3. QDRANT_URL + QDRANT_API_KEY 环境变量 → HTTP 远程模式 + + 本地文件模式是默认推荐方式,对 ARM/Raspberry Pi 友好。 + """ + if memory: + logger.debug("Qdrant 内存模式") + return QdrantClient(location=":memory:") + + # 远程模式: 仅在 QDRANT_URL 为非 localhost 且显式指定 path 为 None 时使用 + remote_url = os.environ.get("QDRANT_URL", "") + if (remote_url + and remote_url.startswith("http") + and "localhost" not in remote_url + and "127.0.0.1" not in remote_url): + api_key = os.environ.get("QDRANT_API_KEY") or None + logger.info("Qdrant 远程模式: %s", remote_url) + return QdrantClient(url=remote_url, api_key=api_key, timeout=10) + + # 默认本地文件模式 + use_path = path or str(DEFAULT_STORAGE_PATH) + logger.info("Qdrant 本地文件模式: %s", use_path) + return QdrantClient(path=use_path) + + +class VectorStore: + """Qdrant 向量知识库封装。 + + 线程不安全,批处理串行使用即可。 + """ + + def __init__( + self, + client: QdrantClient, + collection_name: str | None = None, + vector_dim: int = DEFAULT_VECTOR_DIM, + ) -> None: + self._c = client + config = _load_qdrant_config() + self.collection_name = collection_name or config.get("collection", DEFAULT_COLLECTION) + self.vector_dim = vector_dim + + # ------------------------------------------------------------------ # + # Collection 管理 + # ------------------------------------------------------------------ # + + def init_collection(self, *, recreate: bool = False) -> None: + """创建 collection(已存在时若 recreate 则重建)。""" + exists = self._c.collection_exists(self.collection_name) + if exists and not recreate: + logger.debug("Collection %s 已存在,跳过初始化", self.collection_name) + return + if exists and recreate: + logger.warning("重建 collection %s", self.collection_name) + self._c.delete_collection(self.collection_name) + + self._c.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams( + size=self.vector_dim, + distance=DEFAULT_DISTANCE, + ), + ) + logger.info( + "已创建 collection %s (dim=%d distance=%s)", + self.collection_name, self.vector_dim, DEFAULT_DISTANCE.name, + ) + + def info(self) -> CollectionInfo: + """获取 collection 概览信息。""" + exists = self._c.collection_exists(self.collection_name) + if not exists: + return CollectionInfo(name=self.collection_name, exists=False) + c_info = self._c.get_collection(self.collection_name) + return CollectionInfo( + name=self.collection_name, + exists=True, + vectors_count=c_info.points_count or 0, + indexed_vectors_count=getattr(c_info, "indexed_vectors_count", None), + segments_count=getattr(c_info, "segments_count", None), + ) + + def count(self) -> int: + """向量总数。""" + try: + return self._c.count(self.collection_name).count + except Exception: + return 0 + + # ------------------------------------------------------------------ # + # 数据写入(幂等 upsert) + # ------------------------------------------------------------------ # + + def upsert( + self, + points: list[dict], + *, + batch_size: int = 100, + ) -> int: + """批量幂等写入。 + + Args: + points: 每个 dict 包含: + id (str) point ID(url_hash) + vector (list[float]) 嵌入向量 + payload (dict) 任意结构化数据 + batch_size: 每批写入条数 + + Returns: + 写入条数 + """ + structs = [ + PointStruct( + id=url_hash_to_uuid(p["id"]), + vector=p["vector"], + payload={"url_hash": p["id"], **(p.get("payload") or {})}, + ) + for p in points + ] + total = len(structs) + for i in range(0, total, batch_size): + chunk = structs[i : i + batch_size] + self._c.upsert(collection_name=self.collection_name, points=chunk) + logger.debug( + "upsert 批 %d/%d (%d 条)", + i // batch_size + 1, (total + batch_size - 1) // batch_size, len(chunk), + ) + logger.info("upsert 完成: %d 条 → collection %s", total, self.collection_name) + return total + + # ------------------------------------------------------------------ # + # 检索 + # ------------------------------------------------------------------ # + + def query( + self, + query_vector: list[float], + *, + top_k: int = 10, + search_filter: SearchFilter | None = None, + score_threshold: float | None = None, + ) -> list[SearchResult]: + """语义检索 + 可选结构化过滤。 + + Args: + query_vector: 嵌入向量(需与 collection 维度一致) + top_k: 返回条数 + search_filter: 结构化过滤(AND 关系) + score_threshold: 最低余弦相似度 + + Returns: + 列表按 score 降序 + """ + q_filter = _build_filter(search_filter) + hits = self._c.query_points( + collection_name=self.collection_name, + query=query_vector, + query_filter=q_filter, + limit=top_k, + score_threshold=score_threshold, + with_payload=True, + with_vectors=False, + ) + results: list[SearchResult] = [] + for p in hits.points: + payload = p.payload or {} + results.append(SearchResult( + url_hash=payload.get("url_hash") or "", + score=p.score if p.score is not None else 0.0, + title=payload.get("title") or "", + title_zh=payload.get("title_zh") or "", + url=payload.get("url") or "", + source_id=payload.get("source_id") or "", + publish_time=payload.get("publish_time") or "", + events=payload.get("events") or [], + content_zh_preview=payload.get("content_zh_preview") or "", + )) + logger.debug("检索完成 top_k=%d → %d 条", top_k, len(results)) + return results + + def close(self) -> None: + self._c.close() + + def __enter__(self) -> "VectorStore": + return self + + def __exit__(self, *_: object) -> None: + self.close() + + +# --------------------------------------------------------------------------- # +# Filter 构建 +# --------------------------------------------------------------------------- # + + +def _build_filter(f: SearchFilter | None) -> Filter | None: + """把 SearchFilter 转换为 Qdrant Filter。""" + if f is None: + return None + conditions: list[FieldCondition] = [] + + if f.source_id: + conditions.append( + FieldCondition(key="source_id", match=MatchValue(value=f.source_id)) + ) + if f.source_ids: + conditions.append( + FieldCondition(key="source_id", match=MatchAny(any=f.source_ids)) + ) + if f.stock_codes: + conditions.append( + FieldCondition( + key="events[].stock_codes", + match=MatchAny(any=f.stock_codes), + ) + ) + if f.sentiment: + conditions.append( + FieldCondition( + key="events[].sentiment", + match=MatchValue(value=f.sentiment), + ) + ) + if f.importance_min is not None: + conditions.append( + FieldCondition( + key="events[].importance", + range=Range(gte=f.importance_min), + ) + ) + if f.event_types: + conditions.append( + FieldCondition( + key="events[].event_type", + match=MatchAny(any=f.event_types), + ) + ) + if f.publish_date_from or f.publish_date_to: + try: + range_kwargs: dict = {} + if f.publish_date_from: + range_kwargs["gte"] = f.publish_date_from + "T00:00:00" + if f.publish_date_to: + range_kwargs["lte"] = f.publish_date_to + "T23:59:59" + conditions.append(FieldCondition( + key="publish_time", + range=DatetimeRange(**range_kwargs), + )) + except ValueError: + logger.warning( + "filter 日期格式错误 from=%r to=%r", + f.publish_date_from, f.publish_date_to, + ) + + if not conditions: + return None + return Filter(must=conditions) diff --git a/vectorstore/models.py b/vectorstore/models.py new file mode 100644 index 0000000..584c9a3 --- /dev/null +++ b/vectorstore/models.py @@ -0,0 +1,52 @@ +"""Qdrant 向量存储数据模型 (M6)。""" + +from typing import Any + +from pydantic import BaseModel, Field + + +class SearchFilter(BaseModel): + """可选检索过滤条件,全部为 AND 关系。""" + + source_id: str | None = None + source_ids: list[str] | None = None + stock_codes: list[str] | None = Field(default=None, description="match any") + sentiment: str | None = None # positive / neutral / negative + importance_min: int | None = None # >= N + event_types: list[str] | None = None # match any + publish_date_from: str | None = None # YYYY-MM-DD + publish_date_to: str | None = None # YYYY-MM-DD + + +class SearchResult(BaseModel): + """单条检索结果(含双语信息)。""" + + url_hash: str + score: float + title: str = "" + title_zh: str = "" + url: str = "" + source_id: str = "" + publish_time: str = "" + events: list[dict[str, Any]] = Field(default_factory=list) + content_zh_preview: str = "" # 中文正文前 300 字 + + def short_summary(self) -> str: + codes = set() + for ev in self.events: + codes.update(ev.get("stock_codes", [])) + codes_str = ",".join(sorted(codes)[:5]) or "-" + return ( + f"[{self.source_id}] score={self.score:.4f} " + f"《{self.title_zh[:40] or self.title[:40]}》 {codes_str}" + ) + + +class CollectionInfo(BaseModel): + """Collection 概览信息。""" + + name: str + exists: bool + vectors_count: int = 0 + indexed_vectors_count: int | None = None + segments_count: int | None = None diff --git a/vectorstore/pipeline.py b/vectorstore/pipeline.py new file mode 100644 index 0000000..d8d3cfd --- /dev/null +++ b/vectorstore/pipeline.py @@ -0,0 +1,216 @@ +"""Qdrant 入库管道 + 语义搜索。 + +输入: data/embeddings/{YYYYMMDD}/{url_hash}.json(M5 向量) + data/events/(M4 元数据) +动作: upsert 到 Qdrant collection +搜索: embed query → Qdrant query → 返回 SearchResult 列表 +""" + +import json +import logging +from datetime import datetime +from pathlib import Path + +from crawler.utils import get_news_day +from embedding.client import ( + embed_batch, + load_embedding_config, + make_embedding_client, +) +from vectorstore.client import VectorStore, make_qdrant_client +from vectorstore.models import SearchFilter, SearchResult + +logger = logging.getLogger(__name__) + + +def _load_embedding_files(date_str: str) -> list[dict]: + """加载指定日期的嵌入向量文件(含对应的 M4 元数据)。 + + Args: + date_str: 日期 YYYYMMDD + + Returns: + dict 列表,含 url_hash / vector / article 信息 + """ + embedding_dir = Path(f"data/embeddings/{date_str}") + event_dir = Path(f"data/events/{date_str}") + + if not embedding_dir.exists(): + return [] + + items: list[dict] = [] + for emb_file in sorted(embedding_dir.glob("*.json")): + if emb_file.name == "index.json": + continue + try: + emb_data = json.loads(emb_file.read_text(encoding="utf-8")) + + # 加载对应的 M4 事件文章获取元数据 + event_file = event_dir / emb_file.name + article_data = {} + if event_file.exists(): + article_data = json.loads(event_file.read_text(encoding="utf-8")) + + items.append({ + "url_hash": emb_data["url_hash"], + "source_id": emb_data.get("source_id", ""), + "vector": emb_data["vector"], + "article": article_data, + }) + except (json.JSONDecodeError, Exception) as e: + logger.warning("加载嵌入文件失败 %s: %s", emb_file, e) + + return items + + +def _build_payload(article_data: dict) -> dict: + """从 M4 文章数据构建 Qdrant payload。 + + Args: + article_data: EnTranslatedArticle 的 dict + + Returns: + payload dict + """ + content_zh = article_data.get("content_zh", "") + return { + "title": article_data.get("title", ""), + "title_zh": article_data.get("title_zh", ""), + "url": article_data.get("url", ""), + "source_id": article_data.get("source_id", ""), + "source_name": article_data.get("source_name", ""), + "publish_time": article_data.get("publish_time", ""), + "events": article_data.get("events", []), + "word_count": article_data.get("word_count", 0), + "word_count_zh": article_data.get("word_count_zh", 0), + "content_zh_preview": content_zh[:300] if content_zh else "", + } + + +def ingest_all_embeddings( + date_str: str | None = None, + *, + recreate: bool = False, +) -> dict: + """将所有 M5 向量入库 Qdrant。 + + Args: + date_str: 日期 YYYYMMDD,默认当前新闻日 + recreate: 是否重建 collection + + Returns: + 统计 dict + """ + if date_str is None: + date_str = get_news_day() + + logger.info("══════ 开始 Qdrant 入库,日期: %s ══════", date_str) + + items = _load_embedding_files(date_str) + if not items: + logger.warning("嵌入目录无数据: data/embeddings/%s/", date_str) + return {"date": date_str, "total": 0, "ingested": 0, "failed": 0, "elapsed_sec": 0} + + start_time = datetime.now() + + # 构造 Qdrant 客户端 + client = make_qdrant_client() + store = VectorStore(client) + + try: + # 初始化 collection + store.init_collection(recreate=recreate) + + # 构建 points + points: list[dict] = [] + for item in items: + payload = _build_payload(item["article"]) + points.append({ + "id": item["url_hash"], + "vector": item["vector"], + "payload": payload, + }) + + # 批量写入 + ingested = store.upsert(points) + failed = len(points) - ingested + + finally: + store.close() + + elapsed = (datetime.now() - start_time).total_seconds() + + logger.info( + "══════ Qdrant 入库完成: %d 条,耗时 %.1f 秒 ══════", + ingested, elapsed, + ) + + return { + "date": date_str, + "total": len(items), + "ingested": ingested, + "failed": failed, + "elapsed_sec": elapsed, + } + + +def search_news( + query: str, + *, + top_k: int = 10, + search_filter: SearchFilter | None = None, + score_threshold: float | None = None, +) -> list[SearchResult]: + """语义搜索新闻。 + + 流程: + 1. 将查询文本向量化(使用 M5 Embedding 服务) + 2. Qdrant 语义检索 + + Args: + query: 中文搜索查询 + top_k: 返回条数 + search_filter: 可选过滤条件 + score_threshold: 最低相似度阈值 + + Returns: + SearchResult 列表 + """ + # 1. 向量化查询 + emb_config = load_embedding_config() + emb_client = make_embedding_client(emb_config) + + try: + vectors = embed_batch(emb_client, emb_config, [query]) + if not vectors: + logger.error("查询向量化失败") + return [] + query_vector = vectors[0] + finally: + emb_client.close() + + # 2. Qdrant 检索 + qdrant = make_qdrant_client() + store = VectorStore(qdrant) + + try: + results = store.query( + query_vector=query_vector, + top_k=top_k, + search_filter=search_filter, + score_threshold=score_threshold, + ) + finally: + store.close() + + return results + + +def get_collection_info() -> dict: + """获取 Qdrant collection 信息。""" + client = make_qdrant_client() + store = VectorStore(client) + try: + info = store.info() + return info.model_dump() + finally: + store.close()