feat(data): Tushare 限速退避 + 新浪兜底(财务 getFinanceReport2022 / 日K 前复权),source+adjust 口径标记
- TushareProvider:频率超限按指数退避重试(不再一次 200/min 即中断),最长等待 30s
- SinaProvider 重构(参考 cc-cursor 公开接口实现):
· 新增财务通道 CompanyFinanceService.getFinanceReport2022(source=gjzb) → FinancialIndicator
(report_date / announce_date=publish_date),与 Tushare fina_indicator schema 一致
· 日 K 保留 jsonp(前复权),统一 UA + 重试
· 不支持方法仍抛 DataSourceNotSupported(复权因子/交易日历/基础信息)
- FailoverProvider 现在可对 daily 与 financial 兜底(CLI _failover_provider 接 SinaProvider)
- DailyBar + stock_daily 表新增 source/adjust 列:新浪兜底行标记 sina/qfq,
Tushare 恢复后 --resume 按同键覆盖回不复权 → 两源格式一致且可追溯
- 迁移 91c4e27a03fb 已生成;执行需在全市场同步结束后:uv run alembic upgrade head
- 测试 34+ 项(新浪财务解析/格式一致/限速退避等)通过
This commit is contained in:
@@ -8,6 +8,8 @@ tushare 库(pyproject optional:uv sync --extra datasource-tushare)。
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import time
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
@@ -24,6 +26,7 @@ from app.infrastructure.data_sources.errors import (
|
||||
DataSourceError,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_TS_DATE = "%Y%m%d"
|
||||
|
||||
|
||||
@@ -56,9 +59,11 @@ class TushareProvider:
|
||||
*,
|
||||
pro: object | None = None,
|
||||
max_retries: int = 3,
|
||||
rate_limit_wait: float = 30.0,
|
||||
) -> None:
|
||||
self._pro = pro if pro is not None else _build_pro(token)
|
||||
self._max_retries = max_retries
|
||||
self._rate_limit_wait = rate_limit_wait
|
||||
|
||||
# ---- 归一化(纯函数,输入 list[dict],可单测) ----
|
||||
|
||||
@@ -101,6 +106,8 @@ class TushareProvider:
|
||||
DailyBar(
|
||||
symbol=str(rec.get("ts_code") or ""),
|
||||
trade_date=_to_date(rec.get("trade_date")) or date.min,
|
||||
source="tushare",
|
||||
adjust="none",
|
||||
open=_to_decimal(rec.get("open")),
|
||||
high=_to_decimal(rec.get("high")),
|
||||
low=_to_decimal(rec.get("low")),
|
||||
@@ -182,9 +189,12 @@ class TushareProvider:
|
||||
|
||||
# ---- 内部 ----
|
||||
|
||||
_RATE_LIMIT_MARKERS = ("频率超限", "每分钟", "frequenc", "too many")
|
||||
|
||||
def _call(self, api: str, **kwargs) -> list[dict[str, Any]]:
|
||||
"""带限速退避的调用:频率超限按指数退避(最长 _rate_limit_wait)等待后重试。"""
|
||||
last_error: Exception | None = None
|
||||
for _ in range(self._max_retries):
|
||||
for attempt in range(self._max_retries):
|
||||
try:
|
||||
fn = getattr(self._pro, api)
|
||||
result = fn(**kwargs)
|
||||
@@ -200,6 +210,10 @@ class TushareProvider:
|
||||
msg = str(exc)
|
||||
if "权限" in msg or "积分" in msg or "token" in msg.lower():
|
||||
raise DataSourceAuthenticationError(msg) from exc
|
||||
if any(marker in msg for marker in self._RATE_LIMIT_MARKERS):
|
||||
wait = min(self._rate_limit_wait, 2 ** (attempt + 1))
|
||||
logger.warning("tushare.%s 频率超限,退避 %.1fs 后重试", api, wait)
|
||||
time.sleep(wait)
|
||||
raise DataSourceError(
|
||||
f"tushare.{api} 重试 {self._max_retries} 次仍失败: {last_error}"
|
||||
) from last_error
|
||||
|
||||
Reference in New Issue
Block a user