simon/news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 大模型使用场景化配置与去重多源记录

Browse Source 
- 新增 configs/llm_models.yaml: 4 个场景(event_extraction/daily_report/stock_report/embedding)
  可独立配置 provider/model/api_key_env/base_url_env/temperature 等,含用途与模型要求说明
- 新增 configs/loader.py: YAML 场景加载器(优先级: CLI 参数 > YAML > .env > 内置默认)
- llm/client.py: load_llm_config 支持 scene 参数,LLMConfig 增加 max_attempts
- embedding/factory+remote+local: provider/model/batch_limit 支持场景覆盖
- scheduler/reporter+stock_reporter: 日报/个股摘要接入场景配置
- dedup: Fingerprint.source_ids 多源记录 + 旧库自动迁移 + DedupResult 多源字段
- scripts/run_dedup: uniques JSON 的 sources 字段 + data/deduped/{day}/sources.json 汇总
- scripts/run_event_extraction: 接入 event_extraction 场景
- 补充测试: 场景优先级/零值、多源合并、旧库迁移、embedding 场景覆盖
This commit is contained in:
simon
2026-08-12 07:57:10 +08:00
parent 0c032196d2
commit 3c65701449
21 changed files with 886 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
embedding/factory.py
+14 -3
View File
@@ -1,9 +1,14 @@
"""Embedding provider 工厂:根据环境变量构造合适后端。"""
"""Embedding provider 工厂:根据配置构造合适后端。
配置优先级: 显式参数 > configs/llm_models.yaml scenes.embedding > .env > 默认。
"""
from __future__ import annotations
import os
from configs.loader import load_scene_config
from .base import AsyncEmbeddingProvider, EmbeddingProvider
from .models import EmbeddingError, EmbeddingProviderType
from .remote import (
@@ -20,14 +25,20 @@ def _read_env(key: str, default: str | None = None) -> str | None:
def resolve_provider_type(provider: str | None = None) -> EmbeddingProviderType:
"""根据 provider 参数 / env 解析出 EmbeddingProviderType。
"""根据 provider 参数 / YAML 场景 / env 解析出 EmbeddingProviderType。
映射:
dashscope / qwen / remote -> DASHSCOPE
local / local-bge / bge / bge-m3 -> LOCAL_BGE
默认 dashscope。
"""
p = (provider or _read_env("EMBEDDING_PROVIDER", "dashscope") or "dashscope").lower()
scene_provider = load_scene_config("embedding").get("provider")
p = (
provider
or scene_provider
or _read_env("EMBEDDING_PROVIDER", "dashscope")
or "dashscope"
).lower()
if p in ("dashscope", "qwen", "remote"):
return EmbeddingProviderType.DASHSCOPE
if p in ("local", "local-bge", "bge", "bge-m3"):
embedding/local.py
+5 -1
View File
@@ -21,6 +21,8 @@ from .models import EmbeddingError
if TYPE_CHECKING:
from sentence_transformers import SentenceTransformer
from configs.loader import load_scene_config
LOCAL_DEFAULT_MODEL = "BAAI/bge-m3"
LOCAL_DEFAULT_DIM = 1024
@@ -56,10 +58,12 @@ class LocalBGEEmbeddingProvider(EmbeddingProvider):
normalize: bool = True,
) -> None:
st_cls = _try_import_st()
# 模型优先级:CLI/参数 > LOCAL_EMBEDDING_MODEL > 默认值
# 模型优先级:CLI/参数 > YAML scenes.embedding > LOCAL_EMBEDDING_MODEL > 默认值
# 不读全局 EMBEDDING_MODEL,避免与 DashScope 冲突
scene_model = load_scene_config("embedding").get("model")
self.model = (
model
or scene_model
or _read_env("LOCAL_EMBEDDING_MODEL", LOCAL_DEFAULT_MODEL)
or LOCAL_DEFAULT_MODEL
)
embedding/remote.py
+59 -19
View File
@@ -5,10 +5,14 @@
model: text-embedding-v3 (1024 维)
限制: 单次请求 input ≤ 25 条
环境变量:
DASHSCOPE_API_KEY
QWEN_BASE_URL (默认百炼兼容路径)
EMBEDDING_MODEL (默认 text-embedding-v3)
配置来源(优先级从高到低):
1. 构造参数(model / api_key / base_url / max_attempts)
2. configs/llm_models.yaml 的 scenes.embedding
3. 环境变量 / .env:
DASHSCOPE_EMBEDDING_API_KEY / DASHSCOPE_API_KEY
DASHSCOPE_EMBEDDING_BASE_URL / QWEN_BASE_URL
DASHSCOPE_EMBEDDING_MODEL
4. 代码内置默认值(text-embedding-v3 / 1024 维)
"""
from __future__ import annotations
@@ -19,6 +23,8 @@ import os
from loguru import logger
from openai import AsyncOpenAI, OpenAI
from configs.loader import load_scene_config
from .base import AsyncEmbeddingProvider, EmbeddingProvider
from .models import EmbeddingError
@@ -32,6 +38,9 @@ DEFAULT_MAX_ATTEMPTS = 3
RETRY_BASE_WAIT_SEC = 1.0
RETRY_MAX_WAIT_SEC = 8.0
# embedding 场景名(对应 configs/llm_models.yaml scenes.embedding)
SCENE_EMBEDDING = "embedding"
def _read_env(key: str, default: str | None = None) -> str | None:
val = os.environ.get(key)
@@ -40,23 +49,48 @@ def _read_env(key: str, default: str | None = None) -> str | None:
return val.strip()
def _scene() -> dict:
"""读取 YAML embedding 场景配置(不存在时为空 dict)。"""
return load_scene_config(SCENE_EMBEDDING)
def _scene_int(key: str, default: int) -> int:
try:
return int(_scene().get(key) or default)
except (TypeError, ValueError):
return default
def _scene_float(key: str, default: float) -> float:
try:
return float(_scene().get(key) or default)
except (TypeError, ValueError):
return default
def _resolve_config() -> tuple[str, str, str]:
"""读取 API key / base_url / model,返回 (api_key, base_url, model)。
优先级: YAML 场景 > 环境变量 > 内置默认。
模型名优先级:DASHSCOPE_EMBEDDING_MODEL > 默认值。
不再读全局 EMBEDDING_MODEL,避免与 LOCAL provider 冲突。
"""
api_key = _read_env("DASHSCOPE_EMBEDDING_API_KEY") or _read_env("DASHSCOPE_API_KEY") or ""
sc = _scene()
api_key_env = sc.get("api_key_env") or "DASHSCOPE_EMBEDDING_API_KEY"
api_key = _read_env(api_key_env) or _read_env("DASHSCOPE_API_KEY") or ""
if not api_key:
raise EmbeddingError("DASHSCOPE_EMBEDDING_API_KEY 或 DASHSCOPE_API_KEY 未配置")
# DASHSCOPE_EMBEDDING_BASE_URL -> QWEN_BASE_URL(兜底) -> 默认
raise EmbeddingError(
f"{api_key_env} 或 DASHSCOPE_API_KEY 未配置"
)
# YAML base_url_env -> DASHSCOPE_EMBEDDING_BASE_URL -> QWEN_BASE_URL(兜底) -> 默认
base_url = (
_read_env("DASHSCOPE_EMBEDDING_BASE_URL")
_read_env(sc.get("base_url_env") or "DASHSCOPE_EMBEDDING_BASE_URL")
or _read_env("QWEN_BASE_URL")
or DASHSCOPE_DEFAULT_BASE
)
model = (
_read_env("DASHSCOPE_EMBEDDING_MODEL", DASHSCOPE_DEFAULT_MODEL)
sc.get("model")
or _read_env("DASHSCOPE_EMBEDDING_MODEL", DASHSCOPE_DEFAULT_MODEL)
or DASHSCOPE_DEFAULT_MODEL
)
return api_key, base_url, model
@@ -78,24 +112,27 @@ class DashScopeEmbeddingProvider(EmbeddingProvider):
model: str | None = None,
api_key: str | None = None,
base_url: str | None = None,
timeout_sec: float = 60.0,
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
timeout_sec: float | None = None,
max_attempts: int | None = None,
batch_limit: int | None = None,
) -> None:
env_key, env_base, env_model = _resolve_config()
self.model = model or env_model
self.dim = DASHSCOPE_DEFAULT_DIM
self.max_attempts = max_attempts
self.max_attempts = max_attempts or _scene_int("max_attempts", DEFAULT_MAX_ATTEMPTS)
self.batch_limit = batch_limit or _scene_int("batch_limit", DASHSCOPE_BATCH_LIMIT)
timeout = timeout_sec or _scene_float("timeout_sec", 60.0)
self._client = OpenAI(
api_key=api_key or env_key,
base_url=base_url or env_base,
timeout=timeout_sec,
timeout=timeout,
)
def embed_batch(self, texts: list[str]) -> list[list[float]]:
if not texts:
return []
results: list[list[float]] = []
for chunk in _chunked(texts, DASHSCOPE_BATCH_LIMIT):
for chunk in _chunked(texts, self.batch_limit):
results.extend(self._call_with_retry(chunk))
return results
@@ -136,24 +173,27 @@ class DashScopeAsyncEmbeddingProvider(AsyncEmbeddingProvider):
model: str | None = None,
api_key: str | None = None,
base_url: str | None = None,
timeout_sec: float = 60.0,
max_attempts: int = DEFAULT_MAX_ATTEMPTS,
timeout_sec: float | None = None,
max_attempts: int | None = None,
batch_limit: int | None = None,
) -> None:
env_key, env_base, env_model = _resolve_config()
self.model = model or env_model
self.dim = DASHSCOPE_DEFAULT_DIM
self.max_attempts = max_attempts
self.max_attempts = max_attempts or _scene_int("max_attempts", DEFAULT_MAX_ATTEMPTS)
self.batch_limit = batch_limit or _scene_int("batch_limit", DASHSCOPE_BATCH_LIMIT)
timeout = timeout_sec or _scene_float("timeout_sec", 60.0)
self._client = AsyncOpenAI(
api_key=api_key or env_key,
base_url=base_url or env_base,
timeout=timeout_sec,
timeout=timeout,
)
async def embed_batch(self, texts: list[str]) -> list[list[float]]:
if not texts:
return []
results: list[list[float]] = []
for chunk in _chunked(texts, DASHSCOPE_BATCH_LIMIT):
for chunk in _chunked(texts, self.batch_limit):
results.extend(await self._call_with_retry(chunk))
return results