初始化

llm/__init__.py

@@ -0,0 +1,61 @@
+"""LLM 翻译 + 投资事件抽取模块 (M4)。
+
+公共 API:
+    - load_llm_config / make_sync_client / make_async_client
+    - translate_and_extract / translate_and_extract_async
+    - PromptTemplate / parse_translation_json
+    - translate_all_deduped（批量管道）
+    - EnTranslatedArticle / EventExtraction / LLMTranslationOutput / Sentiment
+"""
+
+from llm.client import (
+    LLMConfig,
+    load_llm_config,
+    make_async_client,
+    make_sync_client,
+)
+from llm.extractor import (
+    DEFAULT_MAX_ATTEMPTS,
+    MAX_CONTENT_CHARS,
+    PromptTemplate,
+    parse_translation_json,
+    translate_and_extract,
+    translate_and_extract_async,
+)
+from llm.models import (
+    INTERNATIONAL_EVENT_TYPES,
+    MAX_IMPORTANCE,
+    MIN_IMPORTANCE,
+    EnTranslatedArticle,
+    EventExtraction,
+    LLMCallError,
+    LLMTranslationOutput,
+    Sentiment,
+)
+from llm.pipeline import translate_all_deduped
+
+__all__ = [
+    # 客户端
+    "LLMConfig",
+    "load_llm_config",
+    "make_async_client",
+    "make_sync_client",
+    # 翻译+抽取
+    "DEFAULT_MAX_ATTEMPTS",
+    "MAX_CONTENT_CHARS",
+    "PromptTemplate",
+    "parse_translation_json",
+    "translate_and_extract",
+    "translate_and_extract_async",
+    # 批量管道
+    "translate_all_deduped",
+    # 模型
+    "EnTranslatedArticle",
+    "EventExtraction",
+    "INTERNATIONAL_EVENT_TYPES",
+    "LLMCallError",
+    "LLMTranslationOutput",
+    "MAX_IMPORTANCE",
+    "MIN_IMPORTANCE",
+    "Sentiment",
+]

llm/client.py

@@ -0,0 +1,131 @@
+"""LLM 客户端抽象与工厂。
+
+支持 DeepSeek 和 Qwen（百炼），两者均为 OpenAI 兼容接口，共用 openai SDK。
+
+配置来源:
+    - .env → API Key / Base URL（密钥和地址）
+    - configs/system.yaml → provider / model / timeout / temperature（功能配置）
+"""
+
+import logging
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+from openai import AsyncOpenAI, OpenAI
+
+logger = logging.getLogger(__name__)
+
+# Provider 默认基址
+_DEEPSEEK_DEFAULT_BASE = "https://api.deepseek.com"
+_QWEN_DEFAULT_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+
+# 默认模型
+_DEEPSEEK_DEFAULT_MODEL = "deepseek-chat"
+_QWEN_DEFAULT_MODEL = "qwen-plus"
+
+
+def _load_system_config() -> dict:
+    """加载 configs/system.yaml 中 llm 段配置。"""
+    config_path = Path("configs/system.yaml")
+    if config_path.exists():
+        try:
+            with open(config_path, encoding="utf-8") as f:
+                raw = yaml.safe_load(f)
+            return raw.get("llm", {})
+        except Exception:
+            logger.warning("加载 llm 配置失败，使用空配置")
+    return {}
+
+
+@dataclass
+class LLMConfig:
+    """LLM 调用配置（provider / model / api_key / base_url / 参数）。"""
+
+    provider: str             # "deepseek" / "qwen"
+    model: str
+    api_key: str
+    base_url: str
+    timeout_sec: float = 60.0
+    temperature: float = 0.1
+    max_tokens: int = 8192
+
+    def __post_init__(self) -> None:
+        if not self.api_key:
+            raise ValueError(f"LLM provider={self.provider} 的 API key 为空")
+
+
+def load_llm_config(
+    provider: str | None = None,
+    *,
+    model: str | None = None,
+) -> LLMConfig:
+    """根据配置文件构造 LLMConfig。
+
+    provider 为 None 时读 system.yaml llm.provider，默认 deepseek。
+    model 为 None 时读 system.yaml 中对应 provider 的 model。
+
+    Raises:
+        ValueError: API key 未配置
+    """
+    config = _load_system_config()
+    p = (provider or config.get("provider", "deepseek")).lower()
+
+    if p == "deepseek":
+        api_key = os.environ.get("DEEPSEEK_API_KEY", "")
+        base = os.environ.get("DEEPSEEK_BASE_URL", _DEEPSEEK_DEFAULT_BASE)
+        m = model or config.get("deepseek_model", _DEEPSEEK_DEFAULT_MODEL)
+    elif p in ("qwen", "dashscope"):
+        api_key = os.environ.get("QWEN_API_KEY") or os.environ.get("DASHSCOPE_API_KEY") or ""
+        base = os.environ.get("QWEN_BASE_URL", _QWEN_DEFAULT_BASE)
+        m = model or config.get("qwen_model", _QWEN_DEFAULT_MODEL)
+        p = "qwen"
+    else:
+        raise ValueError(f"未知 LLM provider: {p!r}，仅支持 deepseek / qwen")
+
+    if not api_key:
+        raise ValueError(
+            f"LLM provider={p} 的 API key 未配置，请检查 .env 中的 "
+            f"{'DEEPSEEK_API_KEY' if p == 'deepseek' else 'QWEN_API_KEY'}"
+        )
+
+    timeout = float(config.get("timeout_sec", 60.0))
+    temperature = float(config.get("temperature", 0.1))
+    max_tokens = int(config.get("max_tokens", 8192))
+
+    return LLMConfig(
+        provider=p,
+        model=m,
+        api_key=api_key,
+        base_url=base,
+        timeout_sec=timeout,
+        temperature=temperature,
+        max_tokens=max_tokens,
+    )
+
+
+def make_sync_client(config: LLMConfig) -> OpenAI:
+    """构造同步 OpenAI 客户端（指向 DeepSeek/Qwen 兼容端点）。"""
+    logger.info(
+        "初始化同步 LLM 客户端: provider=%s model=%s base_url=%s",
+        config.provider, config.model, config.base_url,
+    )
+    return OpenAI(
+        api_key=config.api_key,
+        base_url=config.base_url,
+        timeout=config.timeout_sec,
+    )
+
+
+def make_async_client(config: LLMConfig) -> AsyncOpenAI:
+    """构造异步 OpenAI 客户端（用于批处理高并发）。"""
+    logger.info(
+        "初始化异步 LLM 客户端: provider=%s model=%s base_url=%s",
+        config.provider, config.model, config.base_url,
+    )
+    return AsyncOpenAI(
+        api_key=config.api_key,
+        base_url=config.base_url,
+        timeout=config.timeout_sec,
+    )

llm/extractor.py

@@ -0,0 +1,370 @@
+"""LLM 翻译 + 投资事件抽取主流程。
+
+输入: ProcessedArticle（M2/M3 输出）
+输出: EnTranslatedArticle（含中英文双语内容 + 抽取事件 + 元信息）
+
+设计:
+    1. 加载 prompts/translation_and_extraction.md，分离 system/user 模板
+    2. 单次 LLM 调用完成翻译 + 事件抽取（节省 token）
+    3. 解析 JSON → Pydantic LLMTranslationOutput 强校验 + 重试
+    4. 限制正文长度避免触顶 context window
+"""
+
+import asyncio
+import json
+import logging
+import re
+import time
+from pathlib import Path
+
+from openai import AsyncOpenAI, OpenAI
+
+from extractor.models import ProcessedArticle
+from llm.client import LLMConfig
+from llm.models import (
+    EnTranslatedArticle,
+    LLMCallError,
+    LLMTranslationOutput,
+)
+
+logger = logging.getLogger(__name__)
+
+# Prompt 模板路径
+DEFAULT_PROMPT_PATH = Path("prompts/translation_and_extraction.md")
+
+# 文章正文截断长度（保守取 8K 字符，避免超出上下文窗口）
+MAX_CONTENT_CHARS = 8000
+
+# 重试设置
+DEFAULT_MAX_ATTEMPTS = 3
+RETRY_BASE_WAIT_SEC = 1.0
+RETRY_MAX_WAIT_SEC = 8.0
+
+# Prompt 中 system / user 分隔标记
+_SYSTEM_SECTION_START = "## System Prompt"
+_USER_SECTION_START = "## User Input"
+
+
+class PromptTemplate:
+    """Prompt 模板加载器。
+
+    从 prompts/translation_and_extraction.md 读取模板，
+    分离 System Prompt 和 User Input 两部分。
+    User Input 支持 {title} / {source_name} / {publish_time} / {content} 占位符。
+    """
+
+    def __init__(self, template_path: str | Path = DEFAULT_PROMPT_PATH) -> None:
+        self._path = Path(template_path)
+        raw = self._path.read_text(encoding="utf-8")
+
+        # 分离 system 和 user 两部分
+        self._system_prompt, self._user_template = self._parse_template(raw)
+
+    @staticmethod
+    def _parse_template(raw: str) -> tuple[str, str]:
+        """解析模板文件，返回 (system_prompt, user_template)。"""
+        # 找到 ## System Prompt 之后的内容直到 ## User Input
+        sys_match = re.search(
+            r"## System Prompt\s*\n(.*?)(?=---\s*\n## User Input)",
+            raw, re.DOTALL
+        )
+        user_match = re.search(
+            r"## User Input\s*\n(.*)",
+            raw, re.DOTALL
+        )
+
+        system = sys_match.group(1).strip() if sys_match else ""
+        user = user_match.group(1).strip() if user_match else raw
+
+        return system, user
+
+    def render(self, article: ProcessedArticle) -> tuple[str, str]:
+        """渲染 Prompt，返回 (system_prompt, user_prompt)。
+
+        对正文做截断处理。
+        """
+        content = article.content
+        if len(content) > MAX_CONTENT_CHARS:
+            logger.debug(
+                "文章 %s 超长截断: %d → %d",
+                article.url_hash, len(content), MAX_CONTENT_CHARS,
+            )
+            content = content[:MAX_CONTENT_CHARS] + "\n\n[正文过长已截断]"
+
+        user_prompt = (
+            self._user_template
+            .replace("{title}", article.title)
+            .replace("{source_name}", article.source_name)
+            .replace("{publish_time}", article.publish_time or "未知")
+            .replace("{content}", content)
+        )
+
+        return self._system_prompt, user_prompt
+
+
+# --------------------------------------------------------------------------- #
+# JSON 提取（LLM 偶尔会包 ```json 围栏）
+# --------------------------------------------------------------------------- #
+
+
+def _extract_json_object(text: str) -> str:
+    """从 LLM 输出中提取首个 JSON 对象字符串（去围栏 / 取首个 {...}）。"""
+    s = text.strip()
+    if s.startswith("```"):
+        s = s.strip("`")
+        if s.lower().startswith("json"):
+            s = s[4:].lstrip("\n").lstrip()
+        if s.endswith("```"):
+            s = s[:-3]
+    start = s.find("{")
+    if start < 0:
+        return s
+    depth = 0
+    for i in range(start, len(s)):
+        if s[i] == "{":
+            depth += 1
+        elif s[i] == "}":
+            depth -= 1
+            if depth == 0:
+                return s[start : i + 1]
+    return s[start:]
+
+
+def parse_translation_json(raw: str) -> LLMTranslationOutput:
+    """把 LLM 输出文本解析为 LLMTranslationOutput（可能抛 LLMCallError）。"""
+    payload = _extract_json_object(raw)
+    try:
+        obj = json.loads(payload)
+    except json.JSONDecodeError as e:
+        raise LLMCallError(f"JSON 解析失败: {e}") from e
+    if not isinstance(obj, dict):
+        raise LLMCallError(f"JSON 顶层非对象: {type(obj).__name__}")
+    try:
+        return LLMTranslationOutput.model_validate(obj)
+    except Exception as e:
+        raise LLMCallError(f"翻译输出 schema 校验失败: {e}") from e
+
+
+# --------------------------------------------------------------------------- #
+# 词数计算
+# --------------------------------------------------------------------------- #
+
+
+def _count_zh_chars(text: str) -> int:
+    """统计中文文本字数（汉字 + 数字 + 字母混合）。"""
+    # 简单统计：去除空白后长度
+    return len(text.replace(" ", "").replace("\n", "").replace("\r", ""))
+
+
+# --------------------------------------------------------------------------- #
+# 同步 / 异步 LLM 调用
+# --------------------------------------------------------------------------- #
+
+
+def _call_llm_sync(
+    client: OpenAI,
+    config: LLMConfig,
+    system_prompt: str,
+    user_prompt: str,
+) -> tuple[str, dict[str, int | None]]:
+    """同步单次 LLM 调用，返回 (raw_text, usage)。"""
+    resp = client.chat.completions.create(
+        model=config.model,
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ],
+        temperature=config.temperature,
+        max_tokens=config.max_tokens,
+        response_format={"type": "json_object"},
+    )
+    text = resp.choices[0].message.content or ""
+    usage = {
+        "prompt_tokens": getattr(resp.usage, "prompt_tokens", None) if resp.usage else None,
+        "completion_tokens": (
+            getattr(resp.usage, "completion_tokens", None) if resp.usage else None
+        ),
+    }
+    return text, usage
+
+
+async def _call_llm_async(
+    client: AsyncOpenAI,
+    config: LLMConfig,
+    system_prompt: str,
+    user_prompt: str,
+) -> tuple[str, dict[str, int | None]]:
+    """异步单次 LLM 调用。"""
+    resp = await client.chat.completions.create(
+        model=config.model,
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_prompt},
+        ],
+        temperature=config.temperature,
+        max_tokens=config.max_tokens,
+        response_format={"type": "json_object"},
+    )
+    text = resp.choices[0].message.content or ""
+    usage = {
+        "prompt_tokens": getattr(resp.usage, "prompt_tokens", None) if resp.usage else None,
+        "completion_tokens": (
+            getattr(resp.usage, "completion_tokens", None) if resp.usage else None
+        ),
+    }
+    return text, usage
+
+
+# --------------------------------------------------------------------------- #
+# 主入口：翻译 + 事件抽取
+# --------------------------------------------------------------------------- #
+
+
+def translate_and_extract(
+    client: OpenAI,
+    config: LLMConfig,
+    article: ProcessedArticle,
+    *,
+    template: PromptTemplate | None = None,
+    max_attempts: int = DEFAULT_MAX_ATTEMPTS,
+) -> EnTranslatedArticle:
+    """同步翻译 + 事件抽取（单篇文章，带重试）。
+
+    Args:
+        client: OpenAI 同步客户端
+        config: LLM 配置
+        article: 待处理的英文新闻
+        template: Prompt 模板，默认加载 prompts/translation_and_extraction.md
+        max_attempts: 最大重试次数
+
+    Returns:
+        EnTranslatedArticle 含双语内容 + 事件
+
+    Raises:
+        LLMCallError: 所有重试均失败
+    """
+    tpl = template or PromptTemplate()
+    system_prompt, user_prompt = tpl.render(article)
+
+    last_err: Exception | None = None
+    for attempt in range(1, max_attempts + 1):
+        try:
+            raw, usage = _call_llm_sync(client, config, system_prompt, user_prompt)
+            output = parse_translation_json(raw)
+
+            # 检查 content_zh 非空
+            if not output.content_zh.strip():
+                raise LLMCallError("LLM 返回的 content_zh 为空")
+
+            return EnTranslatedArticle(
+                source_id=article.source_id,
+                source_name=article.source_name,
+                url=article.url,
+                url_hash=article.url_hash,
+                title=article.title,
+                title_zh=output.title_zh,
+                content_en=article.content,
+                content_zh=output.content_zh,
+                publish_time=article.publish_time,
+                word_count=article.word_count,
+                word_count_zh=_count_zh_chars(output.content_zh),
+                events=output.events,
+                provider=config.provider,
+                model=config.model,
+                attempts=attempt,
+                prompt_tokens=usage.get("prompt_tokens"),
+                completion_tokens=usage.get("completion_tokens"),
+            )
+
+        except LLMCallError as e:
+            last_err = e
+            logger.warning(
+                "LLM 翻译抽取失败 url=%s 尝试 %d/%d: %s",
+                article.url, attempt, max_attempts, e.reason,
+            )
+        except Exception as e:
+            last_err = e
+            logger.warning(
+                "LLM 调用异常 url=%s 尝试 %d/%d: %s: %s",
+                article.url, attempt, max_attempts, type(e).__name__, e,
+            )
+
+        if attempt < max_attempts:
+            wait = min(RETRY_BASE_WAIT_SEC * (2 ** (attempt - 1)), RETRY_MAX_WAIT_SEC)
+            time.sleep(wait)
+
+    raise LLMCallError(
+        f"LLM 翻译抽取放弃，共 {max_attempts} 次尝试: {last_err}",
+        attempts=max_attempts,
+    )
+
+
+async def translate_and_extract_async(
+    client: AsyncOpenAI,
+    config: LLMConfig,
+    article: ProcessedArticle,
+    *,
+    template: PromptTemplate | None = None,
+    max_attempts: int = DEFAULT_MAX_ATTEMPTS,
+    semaphore: asyncio.Semaphore | None = None,
+) -> EnTranslatedArticle:
+    """异步翻译 + 事件抽取（批处理用），与同步版逻辑等价。"""
+    tpl = template or PromptTemplate()
+    system_prompt, user_prompt = tpl.render(article)
+
+    async def _run() -> EnTranslatedArticle:
+        last_err: Exception | None = None
+        for attempt in range(1, max_attempts + 1):
+            try:
+                raw, usage = await _call_llm_async(client, config, system_prompt, user_prompt)
+                output = parse_translation_json(raw)
+
+                if not output.content_zh.strip():
+                    raise LLMCallError("LLM 返回的 content_zh 为空")
+
+                return EnTranslatedArticle(
+                    source_id=article.source_id,
+                    source_name=article.source_name,
+                    url=article.url,
+                    url_hash=article.url_hash,
+                    title=article.title,
+                    title_zh=output.title_zh,
+                    content_en=article.content,
+                    content_zh=output.content_zh,
+                    publish_time=article.publish_time,
+                    word_count=article.word_count,
+                    word_count_zh=_count_zh_chars(output.content_zh),
+                    events=output.events,
+                    provider=config.provider,
+                    model=config.model,
+                    attempts=attempt,
+                    prompt_tokens=usage.get("prompt_tokens"),
+                    completion_tokens=usage.get("completion_tokens"),
+                )
+
+            except LLMCallError as e:
+                last_err = e
+                logger.warning(
+                    "LLM 翻译抽取失败 url=%s 尝试 %d/%d: %s",
+                    article.url, attempt, max_attempts, e.reason,
+                )
+            except Exception as e:
+                last_err = e
+                logger.warning(
+                    "LLM 调用异常 url=%s 尝试 %d/%d: %s: %s",
+                    article.url, attempt, max_attempts, type(e).__name__, e,
+                )
+
+            if attempt < max_attempts:
+                wait = min(RETRY_BASE_WAIT_SEC * (2 ** (attempt - 1)), RETRY_MAX_WAIT_SEC)
+                await asyncio.sleep(wait)
+
+        raise LLMCallError(
+            f"LLM 翻译抽取放弃，共 {max_attempts} 次尝试: {last_err}",
+            attempts=max_attempts,
+        )
+
+    if semaphore is None:
+        return await _run()
+    async with semaphore:
+        return await _run()

llm/models.py

@@ -0,0 +1,144 @@
+"""LLM 翻译 + 投资事件抽取数据模型 (M4)。
+
+EnTranslatedArticle 是 M4 最终落盘格式，包含中英文双语内容和抽取的事件。
+EventExtraction 是 LLM JSON 输出直接映射，经 Pydantic 强校验。
+"""
+
+import re
+from datetime import datetime
+from enum import StrEnum
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class Sentiment(StrEnum):
+    """事件情绪倾向。"""
+
+    POSITIVE = "positive"  # 利好
+    NEUTRAL = "neutral"    # 中性
+    NEGATIVE = "negative"  # 利空
+
+
+# 国际财经事件类型（LLM Prompt 中展示）
+INTERNATIONAL_EVENT_TYPES: tuple[str, ...] = (
+    "财报披露",
+    "并购收购",
+    "产品发布",
+    "监管政策",
+    "宏观经济",
+    "央行决议",
+    "行业动态",
+    "技术突破",
+    "高管变动",
+    "诉讼法律",
+    "市场异动",
+    "地缘政治",
+    "大宗商品",
+    "外汇波动",
+    "其他",
+)
+
+# 美股代码正则：1-5 个大写字母
+_US_STOCK_RE = re.compile(r"^[A-Z]{1,5}$")
+
+MIN_IMPORTANCE = 1
+MAX_IMPORTANCE = 5
+
+
+class EventExtraction(BaseModel):
+    """LLM 输出的单个事件，直接映射 JSON。"""
+
+    event_type: str = Field(..., description="事件类型，见 INTERNATIONAL_EVENT_TYPES")
+    stock_codes: list[str] = Field(
+        default_factory=list,
+        description="涉及美股代码，如 AAPL、TSLA；无相关股票时为空",
+    )
+    sentiment: Sentiment = Field(..., description="positive/neutral/negative")
+    importance: int = Field(
+        ..., ge=MIN_IMPORTANCE, le=MAX_IMPORTANCE, description="1-5 重要程度"
+    )
+    summary_zh: str = Field(
+        default="",
+        max_length=200,
+        description="一句话中文事件摘要",
+    )
+
+    @field_validator("stock_codes")
+    @classmethod
+    def _validate_stock_codes(cls, v: list[str]) -> list[str]:
+        """剔除非美股代码格式、去重、统一大写。"""
+        cleaned: list[str] = []
+        seen: set[str] = set()
+        for code in v:
+            s = (code or "").strip().upper()
+            if not s or not _US_STOCK_RE.match(s):
+                continue
+            if s not in seen:
+                seen.add(s)
+                cleaned.append(s)
+        return cleaned
+
+    @field_validator("event_type")
+    @classmethod
+    def _normalize_event_type(cls, v: str) -> str:
+        s = (v or "").strip()
+        return s if s else "其他"
+
+
+class LLMTranslationOutput(BaseModel):
+    """LLM 单次调用的完整输出 JSON 映射。"""
+
+    title_zh: str = Field(..., description="中文翻译标题")
+    content_zh: str = Field(..., description="中文翻译正文")
+    events: list[EventExtraction] = Field(
+        default_factory=list, description="提取的投资事件列表"
+    )
+
+
+class EnTranslatedArticle(BaseModel):
+    """M4 最终落盘格式：双语文章 + 抽取事件 + 调用元信息。"""
+
+    # ── 来源标识 ──
+    source_id: str
+    source_name: str
+    url: str
+    url_hash: str
+
+    # ── 双语内容 ──
+    title: str = ""              # 英文原标题
+    title_zh: str = ""           # 中文翻译标题
+    content_en: str = ""         # 英文原文
+    content_zh: str = ""         # 中文翻译
+    publish_time: str = ""       # ISO 8601
+    word_count: int = 0          # 英文词数
+    word_count_zh: int = 0       # 中文译文字数
+
+    # ── 抽取事件 ──
+    events: list[EventExtraction] = Field(default_factory=list)
+
+    # ── 调用元信息 ──
+    provider: str = ""           # deepseek / qwen
+    model: str = ""
+    translated_at: datetime = Field(default_factory=datetime.now)
+    attempts: int = 1            # 实际调用次数（含重试）
+    prompt_tokens: int | None = None
+    completion_tokens: int | None = None
+
+    def short_summary(self) -> str:
+        codes = set()
+        for ev in self.events:
+            codes.update(ev.stock_codes)
+        codes_str = ",".join(sorted(codes)[:5]) or "-"
+        return (
+            f"[{self.source_id}] {self.title[:30]}... "
+            f"→ {len(self.events)}events, stocks: {codes_str}"
+        )
+
+
+class LLMCallError(Exception):
+    """LLM 调用失败（网络 / 解析 / 校验）。"""
+
+    def __init__(self, reason: str, *, attempts: int = 0) -> None:
+        super().__init__(reason)
+        self.reason = reason
+        self.attempts = attempts

llm/pipeline.py

@@ -0,0 +1,238 @@
+"""批量翻译 + 事件抽取管道。
+
+输入:  data/deduped/{YYYYMMDD}/uniques/{url_hash}.json（M3 去重后唯一条目）
+输出:  data/events/{YYYYMMDD}/{url_hash}.json
+"""
+
+import json
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime
+from pathlib import Path
+
+import yaml
+
+from crawler.utils import get_news_day
+from extractor.models import ProcessedArticle
+from llm.client import LLMConfig, load_llm_config, make_sync_client
+from llm.extractor import PromptTemplate, translate_and_extract
+from llm.models import EnTranslatedArticle, LLMCallError
+
+logger = logging.getLogger(__name__)
+
+# 默认并发数
+_DEFAULT_CONCURRENCY = 3
+
+
+def _load_concurrency() -> int:
+    """从 system.yaml 读取 LLM 并发配置。"""
+    config_path = Path("configs/system.yaml")
+    if config_path.exists():
+        try:
+            with open(config_path, encoding="utf-8") as f:
+                raw = yaml.safe_load(f)
+            return int(raw.get("llm", {}).get("concurrency", _DEFAULT_CONCURRENCY))
+        except Exception:
+            pass
+    return _DEFAULT_CONCURRENCY
+
+
+def _load_deduped_articles(
+    date_str: str,
+) -> list[ProcessedArticle]:
+    """加载指定日期的去重后唯一条目。
+
+    Args:
+        date_str: 日期 YYYYMMDD
+
+    Returns:
+        ProcessedArticle 列表
+    """
+    base_dir = Path(f"data/deduped/{date_str}/uniques")
+    if not base_dir.exists():
+        return []
+
+    articles: list[ProcessedArticle] = []
+    for json_file in sorted(base_dir.glob("*.json")):
+        try:
+            data = json.loads(json_file.read_text(encoding="utf-8"))
+            articles.append(ProcessedArticle(**data))
+        except (json.JSONDecodeError, Exception) as e:
+            logger.warning("解析去重文章失败 %s: %s", json_file, e)
+
+    return articles
+
+
+def _process_one(
+    article: ProcessedArticle,
+    client,
+    config: LLMConfig,
+    template: PromptTemplate,
+) -> EnTranslatedArticle | None:
+    """处理单篇文章的翻译 + 事件抽取。返回 None 表示失败。"""
+    try:
+        return translate_and_extract(
+            client=client,
+            config=config,
+            article=article,
+            template=template,
+        )
+    except LLMCallError as e:
+        logger.error(
+            "翻译抽取最终失败 url=%s: %s", article.url, e.reason
+        )
+        return None
+    except Exception as e:
+        logger.exception("翻译抽取未预期异常 url=%s: %s", article.url, e)
+        return None
+
+
+def translate_all_deduped(
+    date_str: str | None = None,
+    *,
+    provider: str | None = None,
+    model: str | None = None,
+    concurrency: int | None = None,
+) -> dict:
+    """对去重后的所有唯一条目执行翻译 + 事件抽取。
+
+    Args:
+        date_str: 日期 YYYYMMDD，默认当前新闻日
+        provider: LLM provider，默认从 system.yaml 读取
+        model: LLM model，默认从 system.yaml 读取
+        concurrency: 并发数，默认从 system.yaml 读取
+
+    Returns:
+        统计 dict
+    """
+    if date_str is None:
+        date_str = get_news_day()
+
+    concurrency = concurrency or _load_concurrency()
+
+    logger.info("══════ 开始翻译+事件抽取，日期: %s，并发: %d ══════",
+                date_str, concurrency)
+
+    # 加载去重后的文章
+    articles = _load_deduped_articles(date_str)
+    if not articles:
+        logger.warning("去重目录无文章: data/deduped/%s/uniques/", date_str)
+        return {"date": date_str, "total": 0, "success": 0, "failed": 0, "elapsed_sec": 0}
+
+    # 初始化 LLM 客户端
+    config = load_llm_config(provider=provider, model=model)
+    client = make_sync_client(config)
+    template = PromptTemplate()
+
+    # 输出目录
+    out_dir = Path(f"data/events/{date_str}")
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    start_time = datetime.now()
+    success = 0
+    failed = 0
+
+    # 增量：跳过已翻译的文章
+    new_articles = []
+    skipped = 0
+    for a in articles:
+        if (out_dir / f"{a.url_hash}.json").exists():
+            skipped += 1
+        else:
+            new_articles.append(a)
+    if skipped > 0:
+        logger.info("增量跳过 %d 篇已翻译，剩余 %d 篇待处理", skipped, len(new_articles))
+    articles = new_articles
+
+    # 并发处理
+    with ThreadPoolExecutor(max_workers=concurrency) as executor:
+        futures = {
+            executor.submit(_process_one, article, client, config, template): article
+            for article in articles
+        }
+
+        for future in as_completed(futures):
+            article = futures[future]
+            try:
+                result = future.result()
+            except Exception as e:
+                logger.error("并发任务异常 url=%s: %s", article.url, e)
+                failed += 1
+                continue
+
+            if result is None:
+                failed += 1
+                continue
+
+            # 写入输出文件
+            out_file = out_dir / f"{result.url_hash}.json"
+            out_file.write_text(
+                result.model_dump_json(indent=2, ensure_ascii=False),
+                encoding="utf-8",
+            )
+            success += 1
+            logger.info(
+                "[%s] ✅ %s → %s (%d events, %d zh chars)",
+                result.source_id,
+                result.title[:40],
+                result.title_zh[:30],
+                len(result.events),
+                result.word_count_zh,
+            )
+
+    elapsed = (datetime.now() - start_time).total_seconds()
+
+    # 写入事件索引
+    _write_event_index(date_str, success, failed, elapsed, config)
+
+    logger.info(
+        "══════ 翻译+事件抽取完成: 成功 %d / 失败 %d / 总计 %d，耗时 %.1f 秒 ══════",
+        success, failed, len(articles), elapsed,
+    )
+
+    return {
+        "date": date_str,
+        "total": len(articles),
+        "success": success,
+        "failed": failed,
+        "elapsed_sec": elapsed,
+        "provider": config.provider,
+        "model": config.model,
+    }
+
+
+def _write_event_index(
+    date_str: str,
+    success: int,
+    failed: int,
+    elapsed_sec: float,
+    config: LLMConfig,
+) -> None:
+    """写入事件索引文件。
+
+    Args:
+        date_str: 日期
+        success: 成功数
+        failed: 失败数
+        elapsed_sec: 耗时（秒）
+        config: LLM 配置
+    """
+    out_dir = Path(f"data/events/{date_str}")
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    index_data = {
+        "date": date_str,
+        "success": success,
+        "failed": failed,
+        "elapsed_sec": round(elapsed_sec, 1),
+        "provider": config.provider,
+        "model": config.model,
+        "generated_at": datetime.now().isoformat(),
+    }
+
+    index_path = out_dir / "index.json"
+    index_path.write_text(
+        json.dumps(index_data, indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    logger.info("事件索引已写入: %s", index_path)