1 Commits
Author SHA1 Message Date
simon 9aa44e610d feat: LLM/Embedding 重试次数纳入 system.yaml 统一配置
- system.yaml: llm.max_retries(死配置) 改为 llm.max_attempts;embedding 段新增 max_attempts
- llm/client.py: LLMConfig 新增 max_attempts(默认 3 兜底),load_llm_config 读取配置
- llm/extractor.py: translate_and_extract(_async) max_attempts=None 时取 config.max_attempts
- embedding/client.py: load_embedding_config 读取 embedding.max_attempts
- scheduler/reporter.py: _call_llm_simple max_retries=None 时取 max_attempts-1
- 新增 2 个配置读取测试;已同步 pi5 验证
2026-08-05 08:48:43 +08:00
7 changed files with 37 additions and 7 deletions
+3 -2
View File
@@ -43,10 +43,10 @@ dedup:
# ── LLM 翻译+事件抽取 ────────────────────────────────
llm:
provider: "deepseek"
deepseek_model: "deepseek-chat"
deepseek_model: "deepseek-v4-flash"
qwen_model: "qwen-plus"
timeout_sec: 60
max_retries: 3
max_attempts: 3 # 单篇总尝试次数(含首次),失败后指数退避重试;日报 AI 摘要同用此值
max_tokens: 8192
temperature: 0.1
concurrency: 3
@@ -57,6 +57,7 @@ embedding:
dashscope_model: "text-embedding-v3"
dimension: 1024
batch_size: 10
max_attempts: 3 # 单批总尝试次数(含首次),失败后指数退避重试
timeout_sec: 30
# ── Qdrant ──────────────────────────────────────────
+2
View File
@@ -92,6 +92,7 @@ def load_embedding_config(
dimension = int(sys_cfg.get("dimension", DASHSCOPE_DEFAULT_DIM))
batch_size = min(int(sys_cfg.get("batch_size", DASHSCOPE_BATCH_LIMIT)), DASHSCOPE_BATCH_LIMIT)
timeout = float(sys_cfg.get("timeout_sec", 30.0))
max_attempts = int(sys_cfg.get("max_attempts", DEFAULT_MAX_ATTEMPTS))
if not api_key:
raise EmbeddingError("DASHSCOPE_API_KEY 未配置,请检查 .env")
@@ -104,6 +105,7 @@ def load_embedding_config(
dimension=dimension,
batch_size=batch_size,
timeout_sec=timeout,
max_attempts=max_attempts,
)
+6
View File
@@ -25,6 +25,9 @@ _QWEN_DEFAULT_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
_DEEPSEEK_DEFAULT_MODEL = "deepseek-chat"
_QWEN_DEFAULT_MODEL = "qwen-plus"
# 默认单次调用总尝试次数(含首次;system.yaml llm.max_attempts 未配置时兜底)
_DEFAULT_MAX_ATTEMPTS = 3
def _load_system_config() -> dict:
"""加载 configs/system.yaml 中 llm 段配置。"""
@@ -50,6 +53,7 @@ class LLMConfig:
timeout_sec: float = 60.0
temperature: float = 0.1
max_tokens: int = 8192
max_attempts: int = _DEFAULT_MAX_ATTEMPTS # 单次调用总尝试次数(含首次)
def __post_init__(self) -> None:
if not self.api_key:
@@ -93,6 +97,7 @@ def load_llm_config(
timeout = float(config.get("timeout_sec", 60.0))
temperature = float(config.get("temperature", 0.1))
max_tokens = int(config.get("max_tokens", 8192))
max_attempts = int(config.get("max_attempts", _DEFAULT_MAX_ATTEMPTS))
return LLMConfig(
provider=p,
@@ -102,6 +107,7 @@ def load_llm_config(
timeout_sec=timeout,
temperature=temperature,
max_tokens=max_tokens,
max_attempts=max_attempts,
)
+8 -3
View File
@@ -226,7 +226,7 @@ def translate_and_extract(
article: ProcessedArticle,
*,
template: PromptTemplate | None = None,
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
max_attempts: int | None = None,
) -> EnTranslatedArticle:
"""同步翻译 + 事件抽取(单篇文章,带重试)。
@@ -235,7 +235,8 @@ def translate_and_extract(
config: LLM 配置
article: 待处理的英文新闻
template: Prompt 模板,默认加载 prompts/translation_and_extraction.md
max_attempts: 最大试次数
max_attempts: 最大试次数(含首次);None 时取 config.max_attempts
(来自 system.yaml llm.max_attempts
Returns:
EnTranslatedArticle 含双语内容 + 事件
@@ -243,6 +244,8 @@ def translate_and_extract(
Raises:
LLMCallError: 所有重试均失败
"""
if max_attempts is None:
max_attempts = config.max_attempts
tpl = template or PromptTemplate()
system_prompt, user_prompt = tpl.render(article)
@@ -305,10 +308,12 @@ async def translate_and_extract_async(
article: ProcessedArticle,
*,
template: PromptTemplate | None = None,
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
max_attempts: int | None = None,
semaphore: asyncio.Semaphore | None = None,
) -> EnTranslatedArticle:
"""异步翻译 + 事件抽取(批处理用),与同步版逻辑等价。"""
if max_attempts is None:
max_attempts = config.max_attempts
tpl = template or PromptTemplate()
system_prompt, user_prompt = tpl.render(article)
+6 -2
View File
@@ -283,7 +283,7 @@ def _call_llm_simple(
system_prompt: str,
user_prompt: str,
max_tokens: int = 600,
max_retries: int = 2,
max_retries: int | None = None,
) -> str:
"""封装 LLM 调用,带重试和客户端复用。
@@ -291,7 +291,8 @@ def _call_llm_simple(
system_prompt: system role 内容
user_prompt: user role 内容
max_tokens: 最大输出 token
max_retries: 最大重试次数不含首次调用
max_retries: 重试次数不含首次None 时取
config.max_attempts - 1system.yaml llm.max_attempts
Returns:
LLM 输出文本所有重试均失败返回空字符串
@@ -308,6 +309,9 @@ def _call_llm_simple(
config = _llm_client_cache["config"]
client = _llm_client_cache[cache_key]
if max_retries is None:
max_retries = max(config.max_attempts - 1, 0)
last_err: str = ""
for attempt in range(1, max_retries + 2): # 首次 + max_retries 次重试
try:
+6
View File
@@ -87,6 +87,12 @@ class TestLoadEmbeddingConfig:
assert cfg.provider == "dashscope"
assert cfg.api_key == "sk-dashscope-test"
def test_max_attempts_from_system_yaml(self, monkeypatch):
"""重试次数取自 system.yaml embedding.max_attempts(当前 3)。"""
monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-dashscope-test")
cfg = load_embedding_config()
assert cfg.max_attempts == 3
def test_fallback_to_qwen_key(self, monkeypatch):
monkeypatch.delenv("DASHSCOPE_API_KEY", raising=False)
monkeypatch.setenv("QWEN_API_KEY", "sk-qwen-key")
+6
View File
@@ -457,6 +457,12 @@ class TestLoadLLMConfig:
with pytest.raises(ValueError, match="API key 未配置"):
load_llm_config(provider="deepseek")
def test_max_attempts_from_system_yaml(self, monkeypatch):
"""重试次数取自 system.yaml llm.max_attempts(当前 3)。"""
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-deepseek-test-key")
config = load_llm_config(provider="deepseek")
assert config.max_attempts == 3
# --------------------------------------------------------------------------- #
# translate_and_extractmock LLM