feat(data): Tushare 限速退避 + 新浪兜底(财务 getFinanceReport2022 / 日K 前复权),source+adjust 口径标记

- TushareProvider:频率超限按指数退避重试(不再一次 200/min 即中断),最长等待 30s
- SinaProvider 重构(参考 cc-cursor 公开接口实现):
  · 新增财务通道 CompanyFinanceService.getFinanceReport2022(source=gjzb) → FinancialIndicator
    (report_date / announce_date=publish_date),与 Tushare fina_indicator schema 一致
  · 日 K 保留 jsonp(前复权),统一 UA + 重试
  · 不支持方法仍抛 DataSourceNotSupported(复权因子/交易日历/基础信息)
- FailoverProvider 现在可对 daily 与 financial 兜底(CLI _failover_provider 接 SinaProvider)
- DailyBar + stock_daily 表新增 source/adjust 列:新浪兜底行标记 sina/qfq,
  Tushare 恢复后 --resume 按同键覆盖回不复权 → 两源格式一致且可追溯
- 迁移 91c4e27a03fb 已生成;执行需在全市场同步结束后:uv run alembic upgrade head
- 测试 34+ 项(新浪财务解析/格式一致/限速退避等)通过
This commit is contained in:
Simon
2026-09-06 20:52:13 +08:00
parent e2741a0236
commit 56254172b3
8 changed files with 314 additions and 35 deletions
+141 -30
View File
@@ -1,39 +1,71 @@
"""新浪财经 Provider —— 备用数据源。
能力边界(AGENT.md §5.2):
- 新浪日 K 接口返回**前复权**数据,口径与 Tushare 不复权不同,
因此本 Provider 只用于「缺失/不可用时的行情参考与交叉验证」,
不得把结果直接并入不复权主时序库(禁止静默混口径)。
- 新浪不提供复权因子 / 财务指标 → 相应方法抛 DataSourceNotSupported。
通道(公开接口方案参考 cc-cursor/finance/data/sources/sina_source.py):
1. 财务:quotes.sina.cn CompanyFinanceService.getFinanceReport2022(source=gjzb,
匿名免费、一次多期),含披露日 publish_date → FinancialIndicator
(symbol / report_date=end_date / announce_date=publish_date),schema 与
Tushare fina_indicator 一致 —— 用于财务兜底(保留防未来函数所需的公告日)。
2. 日 K:quotes.sina.cn getKLineData(jsonp,**前复权**)。新浪无「不复权 + 独立复权
因子」,因此日线兜底行标记 source=sina、adjust=qfq,与主口径区分;Tushare 恢复
后 --resume 会按日覆盖回不复权行。
能力边界(其余接口新浪不支持 → DataSourceNotSupported):
get_stock_basic / get_trade_cal / get_adjust_factor。
"""
from __future__ import annotations
import json
import re
import time
import urllib.parse
import urllib.request
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from app.domain.entities.market import DailyBar
from app.domain.entities.market import DailyBar, FinancialIndicator
from app.infrastructure.data_sources.errors import (
DataSourceError,
DataSourceNotSupported,
)
_UA = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
)
_KLINE_JSONP = (
"https://quotes.sina.cn/cn/api/jsonp_v2.php/var%20data=/CN_MarketDataService"
".getKLineData?symbol={sina_symbol}&scale=240&ma=no&datalen={datalen}"
)
_FIN_BASE = "https://quotes.sina.cn/cn/api/openapi.php/CompanyFinanceService.getFinanceReport2022"
# 新浪「关键指标」中文项名 → 本项目 FinancialIndicator 字段(None 表示已具备/忽略)
_FIN_FIELD_MAP = {
"基本每股收益": "eps",
"净资产收益率(ROE)": "roe",
"加权净资产收益率": "roe",
"销售毛利率": "gross_margin",
"毛利率": "gross_margin",
"营业总收入": "total_revenue",
"净利润": "net_profit",
}
def _to_sina_symbol(symbol: str) -> str:
"""600519.SH -> sh600519;000001.SZ -> sz000001。"""
code, _, suffix = symbol.partition(".")
prefix = {"SH": "sh", "SZ": "sz", "BJ": "bj"}.get(suffix.upper(), "sh")
return f"{prefix}{code}"
"""600519.SH -> sh600519;000001.SZ -> sz000001;无后缀时按规则猜测。"""
code = symbol.strip().upper()
if code.endswith(".SH"):
return "sh" + code[:-3]
if code.endswith(".SZ"):
return "sz" + code[:-3]
if code.endswith(".BJ"):
return "bj" + code[:-3]
if code.startswith(("6", "9")):
return "sh" + code
if code.startswith(("4", "8")):
return "bj" + code
return "sz" + code
def _extract_jsonp(payload: str) -> list[dict[str, Any]]:
@@ -43,23 +75,110 @@ def _extract_jsonp(payload: str) -> list[dict[str, Any]]:
return json.loads(match.group(1))
def _d(value) -> Decimal | None:
if value is None:
return None
try:
return Decimal(str(value))
except (ValueError, TypeError):
return None
def _to_date(value: str) -> date:
"""兼容 20240831 / 2024-08-31 等格式。"""
digits = re.sub(r"\D", "", str(value))[:8]
return datetime.strptime(digits, "%Y%m%d").date()
class SinaProvider:
"""新浪财经备用数据源(仅日线参考 / 交叉验证)。"""
"""新浪财经备用数据源:财务(与 Tushare schema 一致)+ 日线(前复权)。"""
name = "sina"
def __init__(self, *, timeout: float = 10.0, urlopen=urllib.request.urlopen) -> None:
def __init__(
self,
*,
timeout: float = 10.0,
retries: int = 2,
urlopen=urllib.request.urlopen,
) -> None:
self._timeout = timeout
self._retries = retries
self._urlopen = urlopen
# ---- HTTP(统一 UA / 重试) ----
def _open(self, url: str) -> bytes:
req = urllib.request.Request(url, headers={"User-Agent": _UA})
last_error: Exception | None = None
for attempt in range(self._retries):
try:
with self._urlopen(req, timeout=self._timeout) as resp:
return resp.read()
except Exception as exc: # noqa: BLE001
last_error = exc
if attempt < self._retries - 1:
time.sleep(0.5 * (attempt + 1))
raise DataSourceError(f"sina 请求失败: {last_error}") from last_error
# ---- 财务(兜底 Tushare fina_indicator) ----
def get_financial(self, symbol: str) -> list[FinancialIndicator]:
"""新浪关键指标(source=gjzb),含披露日 publish_date → announce_date。"""
params = {
"paperCode": _to_sina_symbol(symbol),
"source": "gjzb",
"type": "0",
"page": "1",
"num": "100",
}
url = f"{_FIN_BASE}?{urllib.parse.urlencode(params)}"
payload = json.loads(self._open(url).decode("utf-8", errors="replace"))
try:
data = payload["result"]["data"]
report_dates = [item["date_value"] for item in data["report_date"]]
except (KeyError, TypeError) as exc:
raise DataSourceError(f"新浪财务返回结构异常({symbol}): {exc}") from exc
rows: list[FinancialIndicator] = []
for rd in report_dates:
entry = data["report_list"].get(rd)
if not entry:
continue
announce = entry.get("publish_date")
if not announce:
continue # 无披露日不可用于研究(防未来函数)
fields: dict[str, Decimal | None] = {
"eps": None,
"roe": None,
"total_revenue": None,
"net_profit": None,
"gross_margin": None,
}
for item in entry.get("data", []):
std = _FIN_FIELD_MAP.get(item.get("item_title", ""))
if std and fields.get(std) is None:
fields[std] = _d(item.get("item_value"))
rows.append(
FinancialIndicator(
symbol=symbol,
report_date=_to_date(str(rd)),
announce_date=_to_date(str(announce)),
eps=fields["eps"],
roe=fields["roe"],
total_revenue=fields["total_revenue"],
net_profit=fields["net_profit"],
gross_margin=fields["gross_margin"],
)
)
return rows
# ---- 日 K(前复权兜底,标记 adjust=qfq) ----
def get_daily(self, symbol: str, start: date, end: date, datalen: int = 320) -> list[DailyBar]:
"""拉取前复权日 K(新浪仅支持最近 datalen 个自然日窗口)。"""
url = _KLINE_JSONP.format(sina_symbol=_to_sina_symbol(symbol), datalen=datalen)
try:
with self._urlopen(url, timeout=self._timeout) as resp:
payload = resp.read().decode("utf-8", errors="replace")
except Exception as exc: # noqa: BLE001
raise DataSourceError(f"sina 请求失败: {exc}") from exc
payload = self._open(url).decode("utf-8", errors="replace")
bars: list[DailyBar] = []
for rec in _extract_jsonp(payload):
@@ -70,6 +189,8 @@ class SinaProvider:
DailyBar(
symbol=symbol,
trade_date=day,
source="sina",
adjust="qfq",
open=_d(rec.get("open")),
high=_d(rec.get("high")),
low=_d(rec.get("low")),
@@ -79,6 +200,8 @@ class SinaProvider:
)
return bars
# ---- 不支持 ----
def get_stock_basic(self):
raise DataSourceNotSupported("新浪不提供股票基础信息列表")
@@ -86,16 +209,4 @@ class SinaProvider:
raise DataSourceNotSupported("新浪不提供交易日历")
def get_adjust_factor(self, symbol, start, end):
raise DataSourceNotSupported("新浪不提供复权因子(返回数据为前复权口径)")
def get_financial(self, symbol):
raise DataSourceNotSupported("新浪不提供财务指标")
def _d(value) -> Decimal | None:
if value is None:
return None
try:
return Decimal(str(value))
except (ValueError, TypeError):
return None
raise DataSourceNotSupported("新浪不提供复权因子(日线接口为前复权口径)")
@@ -8,6 +8,8 @@ tushare 库(pyproject optional:uv sync --extra datasource-tushare)。
from __future__ import annotations
import importlib
import logging
import time
from datetime import date, datetime
from decimal import Decimal
from typing import Any
@@ -24,6 +26,7 @@ from app.infrastructure.data_sources.errors import (
DataSourceError,
)
logger = logging.getLogger(__name__)
_TS_DATE = "%Y%m%d"
@@ -56,9 +59,11 @@ class TushareProvider:
*,
pro: object | None = None,
max_retries: int = 3,
rate_limit_wait: float = 30.0,
) -> None:
self._pro = pro if pro is not None else _build_pro(token)
self._max_retries = max_retries
self._rate_limit_wait = rate_limit_wait
# ---- 归一化(纯函数,输入 list[dict],可单测) ----
@@ -101,6 +106,8 @@ class TushareProvider:
DailyBar(
symbol=str(rec.get("ts_code") or ""),
trade_date=_to_date(rec.get("trade_date")) or date.min,
source="tushare",
adjust="none",
open=_to_decimal(rec.get("open")),
high=_to_decimal(rec.get("high")),
low=_to_decimal(rec.get("low")),
@@ -182,9 +189,12 @@ class TushareProvider:
# ---- 内部 ----
_RATE_LIMIT_MARKERS = ("频率超限", "每分钟", "frequenc", "too many")
def _call(self, api: str, **kwargs) -> list[dict[str, Any]]:
"""带限速退避的调用:频率超限按指数退避(最长 _rate_limit_wait)等待后重试。"""
last_error: Exception | None = None
for _ in range(self._max_retries):
for attempt in range(self._max_retries):
try:
fn = getattr(self._pro, api)
result = fn(**kwargs)
@@ -200,6 +210,10 @@ class TushareProvider:
msg = str(exc)
if "权限" in msg or "积分" in msg or "token" in msg.lower():
raise DataSourceAuthenticationError(msg) from exc
if any(marker in msg for marker in self._RATE_LIMIT_MARKERS):
wait = min(self._rate_limit_wait, 2 ** (attempt + 1))
logger.warning("tushare.%s 频率超限,退避 %.1fs 后重试", api, wait)
time.sleep(wait)
raise DataSourceError(
f"tushare.{api} 重试 {self._max_retries} 次仍失败: {last_error}"
) from last_error