Compare commits
1
Commits
m9-report-db
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9aa44e610d |
+3
-2
@@ -43,10 +43,10 @@ dedup:
|
|||||||
# ── LLM 翻译+事件抽取 ────────────────────────────────
|
# ── LLM 翻译+事件抽取 ────────────────────────────────
|
||||||
llm:
|
llm:
|
||||||
provider: "deepseek"
|
provider: "deepseek"
|
||||||
deepseek_model: "deepseek-chat"
|
deepseek_model: "deepseek-v4-flash"
|
||||||
qwen_model: "qwen-plus"
|
qwen_model: "qwen-plus"
|
||||||
timeout_sec: 60
|
timeout_sec: 60
|
||||||
max_retries: 3
|
max_attempts: 3 # 单篇总尝试次数(含首次),失败后指数退避重试;日报 AI 摘要同用此值
|
||||||
max_tokens: 8192
|
max_tokens: 8192
|
||||||
temperature: 0.1
|
temperature: 0.1
|
||||||
concurrency: 3
|
concurrency: 3
|
||||||
@@ -57,6 +57,7 @@ embedding:
|
|||||||
dashscope_model: "text-embedding-v3"
|
dashscope_model: "text-embedding-v3"
|
||||||
dimension: 1024
|
dimension: 1024
|
||||||
batch_size: 10
|
batch_size: 10
|
||||||
|
max_attempts: 3 # 单批总尝试次数(含首次),失败后指数退避重试
|
||||||
timeout_sec: 30
|
timeout_sec: 30
|
||||||
|
|
||||||
# ── Qdrant ──────────────────────────────────────────
|
# ── Qdrant ──────────────────────────────────────────
|
||||||
|
|||||||
@@ -92,6 +92,7 @@ def load_embedding_config(
|
|||||||
dimension = int(sys_cfg.get("dimension", DASHSCOPE_DEFAULT_DIM))
|
dimension = int(sys_cfg.get("dimension", DASHSCOPE_DEFAULT_DIM))
|
||||||
batch_size = min(int(sys_cfg.get("batch_size", DASHSCOPE_BATCH_LIMIT)), DASHSCOPE_BATCH_LIMIT)
|
batch_size = min(int(sys_cfg.get("batch_size", DASHSCOPE_BATCH_LIMIT)), DASHSCOPE_BATCH_LIMIT)
|
||||||
timeout = float(sys_cfg.get("timeout_sec", 30.0))
|
timeout = float(sys_cfg.get("timeout_sec", 30.0))
|
||||||
|
max_attempts = int(sys_cfg.get("max_attempts", DEFAULT_MAX_ATTEMPTS))
|
||||||
|
|
||||||
if not api_key:
|
if not api_key:
|
||||||
raise EmbeddingError("DASHSCOPE_API_KEY 未配置,请检查 .env")
|
raise EmbeddingError("DASHSCOPE_API_KEY 未配置,请检查 .env")
|
||||||
@@ -104,6 +105,7 @@ def load_embedding_config(
|
|||||||
dimension=dimension,
|
dimension=dimension,
|
||||||
batch_size=batch_size,
|
batch_size=batch_size,
|
||||||
timeout_sec=timeout,
|
timeout_sec=timeout,
|
||||||
|
max_attempts=max_attempts,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,9 @@ _QWEN_DEFAULT_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
|||||||
_DEEPSEEK_DEFAULT_MODEL = "deepseek-chat"
|
_DEEPSEEK_DEFAULT_MODEL = "deepseek-chat"
|
||||||
_QWEN_DEFAULT_MODEL = "qwen-plus"
|
_QWEN_DEFAULT_MODEL = "qwen-plus"
|
||||||
|
|
||||||
|
# 默认单次调用总尝试次数(含首次;system.yaml llm.max_attempts 未配置时兜底)
|
||||||
|
_DEFAULT_MAX_ATTEMPTS = 3
|
||||||
|
|
||||||
|
|
||||||
def _load_system_config() -> dict:
|
def _load_system_config() -> dict:
|
||||||
"""加载 configs/system.yaml 中 llm 段配置。"""
|
"""加载 configs/system.yaml 中 llm 段配置。"""
|
||||||
@@ -50,6 +53,7 @@ class LLMConfig:
|
|||||||
timeout_sec: float = 60.0
|
timeout_sec: float = 60.0
|
||||||
temperature: float = 0.1
|
temperature: float = 0.1
|
||||||
max_tokens: int = 8192
|
max_tokens: int = 8192
|
||||||
|
max_attempts: int = _DEFAULT_MAX_ATTEMPTS # 单次调用总尝试次数(含首次)
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
def __post_init__(self) -> None:
|
||||||
if not self.api_key:
|
if not self.api_key:
|
||||||
@@ -93,6 +97,7 @@ def load_llm_config(
|
|||||||
timeout = float(config.get("timeout_sec", 60.0))
|
timeout = float(config.get("timeout_sec", 60.0))
|
||||||
temperature = float(config.get("temperature", 0.1))
|
temperature = float(config.get("temperature", 0.1))
|
||||||
max_tokens = int(config.get("max_tokens", 8192))
|
max_tokens = int(config.get("max_tokens", 8192))
|
||||||
|
max_attempts = int(config.get("max_attempts", _DEFAULT_MAX_ATTEMPTS))
|
||||||
|
|
||||||
return LLMConfig(
|
return LLMConfig(
|
||||||
provider=p,
|
provider=p,
|
||||||
@@ -102,6 +107,7 @@ def load_llm_config(
|
|||||||
timeout_sec=timeout,
|
timeout_sec=timeout,
|
||||||
temperature=temperature,
|
temperature=temperature,
|
||||||
max_tokens=max_tokens,
|
max_tokens=max_tokens,
|
||||||
|
max_attempts=max_attempts,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+8
-3
@@ -226,7 +226,7 @@ def translate_and_extract(
|
|||||||
article: ProcessedArticle,
|
article: ProcessedArticle,
|
||||||
*,
|
*,
|
||||||
template: PromptTemplate | None = None,
|
template: PromptTemplate | None = None,
|
||||||
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
max_attempts: int | None = None,
|
||||||
) -> EnTranslatedArticle:
|
) -> EnTranslatedArticle:
|
||||||
"""同步翻译 + 事件抽取(单篇文章,带重试)。
|
"""同步翻译 + 事件抽取(单篇文章,带重试)。
|
||||||
|
|
||||||
@@ -235,7 +235,8 @@ def translate_and_extract(
|
|||||||
config: LLM 配置
|
config: LLM 配置
|
||||||
article: 待处理的英文新闻
|
article: 待处理的英文新闻
|
||||||
template: Prompt 模板,默认加载 prompts/translation_and_extraction.md
|
template: Prompt 模板,默认加载 prompts/translation_and_extraction.md
|
||||||
max_attempts: 最大重试次数
|
max_attempts: 最大尝试次数(含首次);None 时取 config.max_attempts
|
||||||
|
(来自 system.yaml llm.max_attempts)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
EnTranslatedArticle 含双语内容 + 事件
|
EnTranslatedArticle 含双语内容 + 事件
|
||||||
@@ -243,6 +244,8 @@ def translate_and_extract(
|
|||||||
Raises:
|
Raises:
|
||||||
LLMCallError: 所有重试均失败
|
LLMCallError: 所有重试均失败
|
||||||
"""
|
"""
|
||||||
|
if max_attempts is None:
|
||||||
|
max_attempts = config.max_attempts
|
||||||
tpl = template or PromptTemplate()
|
tpl = template or PromptTemplate()
|
||||||
system_prompt, user_prompt = tpl.render(article)
|
system_prompt, user_prompt = tpl.render(article)
|
||||||
|
|
||||||
@@ -305,10 +308,12 @@ async def translate_and_extract_async(
|
|||||||
article: ProcessedArticle,
|
article: ProcessedArticle,
|
||||||
*,
|
*,
|
||||||
template: PromptTemplate | None = None,
|
template: PromptTemplate | None = None,
|
||||||
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
|
max_attempts: int | None = None,
|
||||||
semaphore: asyncio.Semaphore | None = None,
|
semaphore: asyncio.Semaphore | None = None,
|
||||||
) -> EnTranslatedArticle:
|
) -> EnTranslatedArticle:
|
||||||
"""异步翻译 + 事件抽取(批处理用),与同步版逻辑等价。"""
|
"""异步翻译 + 事件抽取(批处理用),与同步版逻辑等价。"""
|
||||||
|
if max_attempts is None:
|
||||||
|
max_attempts = config.max_attempts
|
||||||
tpl = template or PromptTemplate()
|
tpl = template or PromptTemplate()
|
||||||
system_prompt, user_prompt = tpl.render(article)
|
system_prompt, user_prompt = tpl.render(article)
|
||||||
|
|
||||||
|
|||||||
@@ -283,7 +283,7 @@ def _call_llm_simple(
|
|||||||
system_prompt: str,
|
system_prompt: str,
|
||||||
user_prompt: str,
|
user_prompt: str,
|
||||||
max_tokens: int = 600,
|
max_tokens: int = 600,
|
||||||
max_retries: int = 2,
|
max_retries: int | None = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""封装 LLM 调用,带重试和客户端复用。
|
"""封装 LLM 调用,带重试和客户端复用。
|
||||||
|
|
||||||
@@ -291,7 +291,8 @@ def _call_llm_simple(
|
|||||||
system_prompt: system role 内容
|
system_prompt: system role 内容
|
||||||
user_prompt: user role 内容
|
user_prompt: user role 内容
|
||||||
max_tokens: 最大输出 token
|
max_tokens: 最大输出 token
|
||||||
max_retries: 最大重试次数(不含首次调用)
|
max_retries: 重试次数(不含首次);None 时取
|
||||||
|
config.max_attempts - 1(system.yaml llm.max_attempts)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
LLM 输出文本;所有重试均失败返回空字符串
|
LLM 输出文本;所有重试均失败返回空字符串
|
||||||
@@ -308,6 +309,9 @@ def _call_llm_simple(
|
|||||||
config = _llm_client_cache["config"]
|
config = _llm_client_cache["config"]
|
||||||
client = _llm_client_cache[cache_key]
|
client = _llm_client_cache[cache_key]
|
||||||
|
|
||||||
|
if max_retries is None:
|
||||||
|
max_retries = max(config.max_attempts - 1, 0)
|
||||||
|
|
||||||
last_err: str = ""
|
last_err: str = ""
|
||||||
for attempt in range(1, max_retries + 2): # 首次 + max_retries 次重试
|
for attempt in range(1, max_retries + 2): # 首次 + max_retries 次重试
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -87,6 +87,12 @@ class TestLoadEmbeddingConfig:
|
|||||||
assert cfg.provider == "dashscope"
|
assert cfg.provider == "dashscope"
|
||||||
assert cfg.api_key == "sk-dashscope-test"
|
assert cfg.api_key == "sk-dashscope-test"
|
||||||
|
|
||||||
|
def test_max_attempts_from_system_yaml(self, monkeypatch):
|
||||||
|
"""重试次数取自 system.yaml embedding.max_attempts(当前 3)。"""
|
||||||
|
monkeypatch.setenv("DASHSCOPE_API_KEY", "sk-dashscope-test")
|
||||||
|
cfg = load_embedding_config()
|
||||||
|
assert cfg.max_attempts == 3
|
||||||
|
|
||||||
def test_fallback_to_qwen_key(self, monkeypatch):
|
def test_fallback_to_qwen_key(self, monkeypatch):
|
||||||
monkeypatch.delenv("DASHSCOPE_API_KEY", raising=False)
|
monkeypatch.delenv("DASHSCOPE_API_KEY", raising=False)
|
||||||
monkeypatch.setenv("QWEN_API_KEY", "sk-qwen-key")
|
monkeypatch.setenv("QWEN_API_KEY", "sk-qwen-key")
|
||||||
|
|||||||
@@ -457,6 +457,12 @@ class TestLoadLLMConfig:
|
|||||||
with pytest.raises(ValueError, match="API key 未配置"):
|
with pytest.raises(ValueError, match="API key 未配置"):
|
||||||
load_llm_config(provider="deepseek")
|
load_llm_config(provider="deepseek")
|
||||||
|
|
||||||
|
def test_max_attempts_from_system_yaml(self, monkeypatch):
|
||||||
|
"""重试次数取自 system.yaml llm.max_attempts(当前 3)。"""
|
||||||
|
monkeypatch.setenv("DEEPSEEK_API_KEY", "sk-deepseek-test-key")
|
||||||
|
config = load_llm_config(provider="deepseek")
|
||||||
|
assert config.max_attempts == 3
|
||||||
|
|
||||||
|
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
# translate_and_extract(mock LLM)
|
# translate_and_extract(mock LLM)
|
||||||
|
|||||||
Reference in New Issue
Block a user