simon
3c65701449
- 新增 configs/llm_models.yaml: 4 个场景(event_extraction/daily_report/stock_report/embedding)
可独立配置 provider/model/api_key_env/base_url_env/temperature 等,含用途与模型要求说明
- 新增 configs/loader.py: YAML 场景加载器(优先级: CLI 参数 > YAML > .env > 内置默认)
- llm/client.py: load_llm_config 支持 scene 参数,LLMConfig 增加 max_attempts
- embedding/factory+remote+local: provider/model/batch_limit 支持场景覆盖
- scheduler/reporter+stock_reporter: 日报/个股摘要接入场景配置
- dedup: Fingerprint.source_ids 多源记录 + 旧库自动迁移 + DedupResult 多源字段
- scripts/run_dedup: uniques JSON 的 sources 字段 + data/deduped/{day}/sources.json 汇总
- scripts/run_event_extraction: 接入 event_extraction 场景
- 补充测试: 场景优先级/零值、多源合并、旧库迁移、embedding 场景覆盖
72 lines
2.3 KiB
Python
72 lines
2.3 KiB
Python
"""Embedding provider 工厂:根据配置构造合适后端。
|
|
|
|
配置优先级: 显式参数 > configs/llm_models.yaml scenes.embedding > .env > 默认。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
from configs.loader import load_scene_config
|
|
|
|
from .base import AsyncEmbeddingProvider, EmbeddingProvider
|
|
from .models import EmbeddingError, EmbeddingProviderType
|
|
from .remote import (
|
|
DashScopeAsyncEmbeddingProvider,
|
|
DashScopeEmbeddingProvider,
|
|
)
|
|
|
|
|
|
def _read_env(key: str, default: str | None = None) -> str | None:
|
|
val = os.environ.get(key)
|
|
if val is None or val.strip() == "":
|
|
return default
|
|
return val.strip()
|
|
|
|
|
|
def resolve_provider_type(provider: str | None = None) -> EmbeddingProviderType:
|
|
"""根据 provider 参数 / YAML 场景 / env 解析出 EmbeddingProviderType。
|
|
|
|
映射:
|
|
dashscope / qwen / remote -> DASHSCOPE
|
|
local / local-bge / bge / bge-m3 -> LOCAL_BGE
|
|
默认 dashscope。
|
|
"""
|
|
scene_provider = load_scene_config("embedding").get("provider")
|
|
p = (
|
|
provider
|
|
or scene_provider
|
|
or _read_env("EMBEDDING_PROVIDER", "dashscope")
|
|
or "dashscope"
|
|
).lower()
|
|
if p in ("dashscope", "qwen", "remote"):
|
|
return EmbeddingProviderType.DASHSCOPE
|
|
if p in ("local", "local-bge", "bge", "bge-m3"):
|
|
return EmbeddingProviderType.LOCAL_BGE
|
|
raise EmbeddingError(f"未知 embedding provider: {provider!r}")
|
|
|
|
|
|
def make_sync_provider(
|
|
provider: str | None = None,
|
|
**kwargs: object,
|
|
) -> EmbeddingProvider:
|
|
"""构造同步 provider。"""
|
|
pt = resolve_provider_type(provider)
|
|
if pt == EmbeddingProviderType.DASHSCOPE:
|
|
return DashScopeEmbeddingProvider(**kwargs) # type: ignore[arg-type]
|
|
# 本地后端
|
|
from .local import LocalBGEEmbeddingProvider
|
|
return LocalBGEEmbeddingProvider(**kwargs) # type: ignore[arg-type]
|
|
|
|
|
|
def make_async_provider(
|
|
provider: str | None = None,
|
|
**kwargs: object,
|
|
) -> AsyncEmbeddingProvider:
|
|
"""构造异步 provider。"""
|
|
pt = resolve_provider_type(provider)
|
|
if pt == EmbeddingProviderType.DASHSCOPE:
|
|
return DashScopeAsyncEmbeddingProvider(**kwargs) # type: ignore[arg-type]
|
|
from .local import LocalBGEAsyncEmbeddingProvider
|
|
return LocalBGEAsyncEmbeddingProvider(**kwargs) # type: ignore[arg-type]
|