feat(backend): Phase 1 数据层 — Domain / Provider / Failover 审计 + 持久化 + 同步 CLI
- domain：市场数据实体（Stock / 交易日历 / 日线 / 复权 / 财务含 announce_date）+ Repository 与 MarketDataProvider Protocol
- 数据源：TushareProvider（归一化、重试、鉴权错误归类）、SinaProvider（备用，明确前复权口径与能力边界）、FailoverProvider + SyncLog 审计（禁止静默切换）
- 持久化：SQLAlchemy 2.x Models + Repository 实现（按业务键幂等 upsert、as_of_date 防未来函数过滤）+ Alembic 迁移
- CLI：uv run python -m app.cli.sync {basic|calendar|daily|financial|verify}，支持 --resume 断点续传
- 真实 Tushare 验证：stock 5556 / 交易日历 366 / daily+factor 242 / 财务 55；sync_log 审计完整
- 测试：38 passed（domain / provider / failover / repository / 未来函数 / 迁移），ruff clean
This commit is contained in:
Simon
@@ -0,0 +1,7 @@
|
||||
"""数据源基础设施：MarketDataProvider 的具体实现与 Failover。
|
||||
|
||||
业务层不直接 import 本目录（AGENT.md §5/§6）——统一经
|
||||
MarketDataProvider（domain.providers）注入；唯一例外是组装处的依赖装配。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,13 @@
|
||||
"""数据源异常类型。"""
|
||||
|
||||
|
||||
class DataSourceError(Exception):
|
||||
"""数据源通用失败（网络、限频、解析等）。"""
|
||||
|
||||
|
||||
class DataSourceAuthenticationError(DataSourceError):
|
||||
"""凭证无效 / 权限不足（如 Tushare token 无该接口权限）。"""
|
||||
|
||||
|
||||
class DataSourceNotSupported(DataSourceError):
|
||||
"""该数据源不提供此能力（如新浪无复权因子），用于 Failover 判定。"""
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Tushare → Sina Failover 包装（AGENT.md §7）。
|
||||
|
||||
规则：
|
||||
- 优先 primary；primary 抛错时才尝试 fallback（避免对空结果做无谓兜底请求）
|
||||
- fallback 不支持该 API（DataSourceNotSupported）或自身失败 → 抛 DataSourceError
|
||||
- 每次尝试都写 SyncLog（source / 成功与否 / 行数 / 区间），禁止静默切换
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from app.domain.entities.market import SyncLog
|
||||
from app.domain.providers import MarketDataProvider
|
||||
from app.infrastructure.data_sources.errors import DataSourceError, DataSourceNotSupported
|
||||
|
||||
|
||||
class FailoverProvider:
|
||||
"""以 primary 为主、fallback 为辅的 MarketDataProvider 实现。"""
|
||||
|
||||
name = "failover"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
primary: MarketDataProvider,
|
||||
fallback: MarketDataProvider | None = None,
|
||||
*,
|
||||
audit: Callable[[SyncLog], None] | None = None,
|
||||
) -> None:
|
||||
self.primary = primary
|
||||
self.fallback = fallback
|
||||
self._audit = audit or (lambda _log: None)
|
||||
|
||||
# ---- 各 API 代理 ----
|
||||
|
||||
def get_stock_basic(self) -> list:
|
||||
return self._with_failover(
|
||||
"get_stock_basic",
|
||||
primary_call=lambda: self.primary.get_stock_basic(),
|
||||
fallback_call=lambda: self.fallback.get_stock_basic(),
|
||||
)
|
||||
|
||||
def get_trade_cal(self, start: date, end: date) -> list:
|
||||
return self._with_failover(
|
||||
"get_trade_cal",
|
||||
start=start,
|
||||
end=end,
|
||||
primary_call=lambda: self.primary.get_trade_cal(start, end),
|
||||
fallback_call=lambda: self.fallback.get_trade_cal(start, end),
|
||||
)
|
||||
|
||||
def get_daily(self, symbol: str, start: date, end: date) -> list:
|
||||
return self._with_failover(
|
||||
"get_daily",
|
||||
start=start,
|
||||
end=end,
|
||||
primary_call=lambda: self.primary.get_daily(symbol, start, end),
|
||||
fallback_call=lambda: self.fallback.get_daily(symbol, start, end),
|
||||
)
|
||||
|
||||
def get_adjust_factor(self, symbol: str, start: date, end: date) -> list:
|
||||
return self._with_failover(
|
||||
"get_adjust_factor",
|
||||
start=start,
|
||||
end=end,
|
||||
primary_call=lambda: self.primary.get_adjust_factor(symbol, start, end),
|
||||
fallback_call=lambda: self.fallback.get_adjust_factor(symbol, start, end),
|
||||
)
|
||||
|
||||
def get_financial(self, symbol: str) -> list:
|
||||
return self._with_failover(
|
||||
"get_financial",
|
||||
primary_call=lambda: self.primary.get_financial(symbol),
|
||||
fallback_call=lambda: self.fallback.get_financial(symbol),
|
||||
)
|
||||
|
||||
# ---- 内部 ----
|
||||
|
||||
def _with_failover(
|
||||
self,
|
||||
api: str,
|
||||
*,
|
||||
primary_call: Callable[[], list],
|
||||
fallback_call: Callable[[], list] | None = None,
|
||||
start: date | None = None,
|
||||
end: date | None = None,
|
||||
) -> list:
|
||||
try:
|
||||
rows = primary_call()
|
||||
except Exception as exc: # noqa: BLE001 —— 统一走审计
|
||||
self._log(
|
||||
source=self.primary.name,
|
||||
api=api,
|
||||
success=False,
|
||||
reason=str(exc),
|
||||
start=start,
|
||||
end=end,
|
||||
)
|
||||
return self._try_fallback(api, fallback_call, start=start, end=end, primary_error=exc)
|
||||
self._log(
|
||||
source=self.primary.name,
|
||||
api=api,
|
||||
success=True,
|
||||
row_count=_len(rows),
|
||||
start=start,
|
||||
end=end,
|
||||
)
|
||||
return rows
|
||||
|
||||
def _try_fallback(self, api, fallback_call, *, start, end, primary_error):
|
||||
if fallback_call is None or self.fallback is None:
|
||||
raise DataSourceError(
|
||||
f"{self.primary.name}.{api} 失败且无备用源: {primary_error}"
|
||||
) from primary_error
|
||||
try:
|
||||
rows = fallback_call()
|
||||
except DataSourceNotSupported as exc:
|
||||
self._log(
|
||||
source=self.fallback.name,
|
||||
api=api,
|
||||
success=False,
|
||||
reason=f"不支持: {exc}",
|
||||
start=start,
|
||||
end=end,
|
||||
)
|
||||
raise DataSourceError(
|
||||
f"{self.primary.name}.{api} 失败，备用源不支持: {primary_error}"
|
||||
) from primary_error
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._log(
|
||||
source=self.fallback.name,
|
||||
api=api,
|
||||
success=False,
|
||||
reason=str(exc),
|
||||
start=start,
|
||||
end=end,
|
||||
)
|
||||
raise DataSourceError(
|
||||
f"主备数据源均失败: primary[{self.primary.name}]={primary_error} "
|
||||
f"fallback[{self.fallback.name}]={exc}"
|
||||
) from exc
|
||||
self._log(
|
||||
source=self.fallback.name,
|
||||
api=api,
|
||||
success=True,
|
||||
row_count=_len(rows),
|
||||
start=start,
|
||||
end=end,
|
||||
)
|
||||
return rows
|
||||
|
||||
def _log(
|
||||
self,
|
||||
*,
|
||||
source: str,
|
||||
api: str,
|
||||
success: bool,
|
||||
reason: str | None = None,
|
||||
row_count: int = 0,
|
||||
start: date | None = None,
|
||||
end: date | None = None,
|
||||
) -> None:
|
||||
self._audit(
|
||||
SyncLog(
|
||||
source=source,
|
||||
api=api,
|
||||
success=success,
|
||||
failure_reason=reason,
|
||||
row_count=row_count,
|
||||
data_start=start,
|
||||
data_end=end,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _len(rows: Any) -> int:
|
||||
return len(rows) if rows is not None else 0
|
||||
@@ -0,0 +1,101 @@
|
||||
"""新浪财经 Provider —— 备用数据源。
|
||||
|
||||
能力边界（AGENT.md §5.2）：
|
||||
- 新浪日 K 接口返回**前复权**数据，口径与 Tushare 不复权不同，
|
||||
因此本 Provider 只用于「缺失/不可用时的行情参考与交叉验证」，
|
||||
不得把结果直接并入不复权主时序库（禁止静默混口径）。
|
||||
- 新浪不提供复权因子 / 财务指标 → 相应方法抛 DataSourceNotSupported。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from app.domain.entities.market import DailyBar
|
||||
from app.infrastructure.data_sources.errors import (
|
||||
DataSourceError,
|
||||
DataSourceNotSupported,
|
||||
)
|
||||
|
||||
_KLINE_JSONP = (
|
||||
"https://quotes.sina.cn/cn/api/jsonp_v2.php/var%20data=/CN_MarketDataService"
|
||||
".getKLineData?symbol={sina_symbol}&scale=240&ma=no&datalen={datalen}"
|
||||
)
|
||||
|
||||
|
||||
def _to_sina_symbol(symbol: str) -> str:
|
||||
"""600519.SH -> sh600519；000001.SZ -> sz000001。"""
|
||||
code, _, suffix = symbol.partition(".")
|
||||
prefix = {"SH": "sh", "SZ": "sz", "BJ": "bj"}.get(suffix.upper(), "sh")
|
||||
return f"{prefix}{code}"
|
||||
|
||||
|
||||
def _extract_jsonp(payload: str) -> list[dict[str, Any]]:
|
||||
match = re.search(r"=\s*(\[.*\])\s*$", payload.strip(), flags=re.DOTALL)
|
||||
if not match:
|
||||
raise DataSourceError("新浪行情返回格式无法解析")
|
||||
return json.loads(match.group(1))
|
||||
|
||||
|
||||
class SinaProvider:
|
||||
"""新浪财经备用数据源（仅日线参考 / 交叉验证）。"""
|
||||
|
||||
name = "sina"
|
||||
|
||||
def __init__(self, *, timeout: float = 10.0, urlopen=urllib.request.urlopen) -> None:
|
||||
self._timeout = timeout
|
||||
self._urlopen = urlopen
|
||||
|
||||
def get_daily(self, symbol: str, start: date, end: date, datalen: int = 320) -> list[DailyBar]:
|
||||
"""拉取前复权日 K（新浪仅支持最近 datalen 个自然日窗口）。"""
|
||||
url = _KLINE_JSONP.format(sina_symbol=_to_sina_symbol(symbol), datalen=datalen)
|
||||
try:
|
||||
with self._urlopen(url, timeout=self._timeout) as resp:
|
||||
payload = resp.read().decode("utf-8", errors="replace")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise DataSourceError(f"sina 请求失败: {exc}") from exc
|
||||
|
||||
bars: list[DailyBar] = []
|
||||
for rec in _extract_jsonp(payload):
|
||||
day = datetime.strptime(rec["day"], "%Y-%m-%d").date()
|
||||
if day < start or day > end:
|
||||
continue
|
||||
bars.append(
|
||||
DailyBar(
|
||||
symbol=symbol,
|
||||
trade_date=day,
|
||||
open=_d(rec.get("open")),
|
||||
high=_d(rec.get("high")),
|
||||
low=_d(rec.get("low")),
|
||||
close=_d(rec.get("close")),
|
||||
volume=_d(rec.get("volume")),
|
||||
)
|
||||
)
|
||||
return bars
|
||||
|
||||
def get_stock_basic(self):
|
||||
raise DataSourceNotSupported("新浪不提供股票基础信息列表")
|
||||
|
||||
def get_trade_cal(self, start, end):
|
||||
raise DataSourceNotSupported("新浪不提供交易日历")
|
||||
|
||||
def get_adjust_factor(self, symbol, start, end):
|
||||
raise DataSourceNotSupported("新浪不提供复权因子（返回数据为前复权口径）")
|
||||
|
||||
def get_financial(self, symbol):
|
||||
raise DataSourceNotSupported("新浪不提供财务指标")
|
||||
|
||||
|
||||
def _d(value) -> Decimal | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(value))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
@@ -0,0 +1,219 @@
|
||||
"""Tushare Provider —— 首选数据源实现。
|
||||
|
||||
依赖注入：pro 客户端（tushare.pro.client 或测试 Fake）。真实运行时惰性加载
|
||||
tushare 库（pyproject optional：uv sync --extra datasource-tushare）。
|
||||
归一化函数只依赖 list[dict]，便于无 pandas 环境下单测。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from app.domain.entities.market import (
|
||||
AdjustFactor,
|
||||
DailyBar,
|
||||
FinancialIndicator,
|
||||
Stock,
|
||||
TradingCalendar,
|
||||
)
|
||||
from app.infrastructure.data_sources.errors import (
|
||||
DataSourceAuthenticationError,
|
||||
DataSourceError,
|
||||
)
|
||||
|
||||
_TS_DATE = "%Y%m%d"
|
||||
|
||||
|
||||
def _to_date(value: str | None) -> date | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
return datetime.strptime(str(value)[:10], _TS_DATE).date()
|
||||
|
||||
|
||||
def _to_decimal(value) -> Decimal | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
num = float(value)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if num != num: # NaN
|
||||
return None
|
||||
return Decimal(str(num))
|
||||
|
||||
|
||||
class TushareProvider:
|
||||
"""封装 Tushare Pro（ts.pro_api）。所有输出已归一化为领域实体。"""
|
||||
|
||||
name = "tushare"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
token: str = "",
|
||||
*,
|
||||
pro: object | None = None,
|
||||
max_retries: int = 3,
|
||||
) -> None:
|
||||
self._pro = pro if pro is not None else _build_pro(token)
|
||||
self._max_retries = max_retries
|
||||
|
||||
# ---- 归一化（纯函数，输入 list[dict]，可单测） ----
|
||||
|
||||
@staticmethod
|
||||
def normalize_stock(records: list[dict[str, Any]]) -> list[Stock]:
|
||||
stocks: list[Stock] = []
|
||||
for rec in records:
|
||||
stocks.append(
|
||||
Stock(
|
||||
symbol=str(rec.get("ts_code") or rec.get("symbol") or ""),
|
||||
name=str(rec.get("name") or ""),
|
||||
industry=rec.get("industry"),
|
||||
area=rec.get("area"),
|
||||
market=rec.get("market"),
|
||||
exchange=rec.get("exchange"),
|
||||
list_date=_to_date(rec.get("list_date")) or date.min,
|
||||
delist_date=_to_date(rec.get("delist_date")),
|
||||
status=str(rec.get("status") or "L"),
|
||||
)
|
||||
)
|
||||
return stocks
|
||||
|
||||
@staticmethod
|
||||
def normalize_calendar(records: list[dict[str, Any]]) -> list[TradingCalendar]:
|
||||
return [
|
||||
TradingCalendar(
|
||||
calendar_date=_to_date(rec.get("cal_date")) or date.min,
|
||||
is_open=bool(rec.get("is_open")),
|
||||
)
|
||||
for rec in records
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def normalize_daily(records: list[dict[str, Any]]) -> list[DailyBar]:
|
||||
bars: list[DailyBar] = []
|
||||
for rec in records:
|
||||
vol = _to_decimal(rec.get("vol"))
|
||||
amount = _to_decimal(rec.get("amount"))
|
||||
bars.append(
|
||||
DailyBar(
|
||||
symbol=str(rec.get("ts_code") or ""),
|
||||
trade_date=_to_date(rec.get("trade_date")) or date.min,
|
||||
open=_to_decimal(rec.get("open")),
|
||||
high=_to_decimal(rec.get("high")),
|
||||
low=_to_decimal(rec.get("low")),
|
||||
close=_to_decimal(rec.get("close")),
|
||||
volume=vol * 100 if vol is not None else None,
|
||||
amount=amount * 1000 if amount is not None else None,
|
||||
)
|
||||
)
|
||||
return bars
|
||||
|
||||
@staticmethod
|
||||
def normalize_adj_factor(records: list[dict[str, Any]]) -> list[AdjustFactor]:
|
||||
return [
|
||||
AdjustFactor(
|
||||
symbol=str(rec.get("ts_code") or ""),
|
||||
trade_date=_to_date(rec.get("trade_date")) or date.min,
|
||||
factor=_to_decimal(rec.get("adj_factor")) or Decimal(1),
|
||||
)
|
||||
for rec in records
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def normalize_financial(records: list[dict[str, Any]]) -> list[FinancialIndicator]:
|
||||
rows: list[FinancialIndicator] = []
|
||||
for rec in records:
|
||||
rows.append(
|
||||
FinancialIndicator(
|
||||
symbol=str(rec.get("ts_code") or ""),
|
||||
report_date=_to_date(rec.get("end_date")) or date.min,
|
||||
announce_date=_to_date(rec.get("ann_date")) or date.min,
|
||||
eps=_to_decimal(rec.get("eps")),
|
||||
roe=_to_decimal(rec.get("roe")),
|
||||
net_profit=_to_decimal(rec.get("n_income_attr_p")),
|
||||
gross_margin=_to_decimal(rec.get("grossprofit_margin")),
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
# ---- 接口调用 ----
|
||||
|
||||
def get_stock_basic(self) -> list[Stock]:
|
||||
records = self._call(
|
||||
"stock_basic",
|
||||
fields="ts_code,symbol,name,area,industry,market,exchange,list_date,delist_date,status",
|
||||
)
|
||||
return self.normalize_stock(records)
|
||||
|
||||
def get_trade_cal(self, start: date, end: date) -> list[TradingCalendar]:
|
||||
records = self._call(
|
||||
"trade_cal",
|
||||
exchange="SSE",
|
||||
start_date=start.strftime(_TS_DATE),
|
||||
end_date=end.strftime(_TS_DATE),
|
||||
is_open="",
|
||||
)
|
||||
return self.normalize_calendar(records)
|
||||
|
||||
def get_daily(self, symbol: str, start: date, end: date) -> list[DailyBar]:
|
||||
records = self._call(
|
||||
"daily",
|
||||
ts_code=symbol,
|
||||
start_date=start.strftime(_TS_DATE),
|
||||
end_date=end.strftime(_TS_DATE),
|
||||
)
|
||||
return self.normalize_daily(records)
|
||||
|
||||
def get_adjust_factor(self, symbol: str, start: date, end: date) -> list[AdjustFactor]:
|
||||
records = self._call(
|
||||
"adj_factor",
|
||||
ts_code=symbol,
|
||||
start_date=start.strftime(_TS_DATE),
|
||||
end_date=end.strftime(_TS_DATE),
|
||||
)
|
||||
return self.normalize_adj_factor(records)
|
||||
|
||||
def get_financial(self, symbol: str) -> list[FinancialIndicator]:
|
||||
records = self._call("fina_indicator", ts_code=symbol)
|
||||
return self.normalize_financial(records)
|
||||
|
||||
# ---- 内部 ----
|
||||
|
||||
def _call(self, api: str, **kwargs) -> list[dict[str, Any]]:
|
||||
last_error: Exception | None = None
|
||||
for _ in range(self._max_retries):
|
||||
try:
|
||||
fn = getattr(self._pro, api)
|
||||
result = fn(**kwargs)
|
||||
if result is None:
|
||||
return []
|
||||
if hasattr(result, "to_dict"):
|
||||
return result.to_dict("records")
|
||||
if isinstance(result, list):
|
||||
return result
|
||||
return []
|
||||
except Exception as exc: # noqa: BLE001 —— tushare 异常无统一类型，逐一归类
|
||||
last_error = exc
|
||||
msg = str(exc)
|
||||
if "权限" in msg or "积分" in msg or "token" in msg.lower():
|
||||
raise DataSourceAuthenticationError(msg) from exc
|
||||
raise DataSourceError(
|
||||
f"tushare.{api} 重试 {self._max_retries} 次仍失败: {last_error}"
|
||||
) from last_error
|
||||
|
||||
|
||||
def _build_pro(token: str):
|
||||
if not token:
|
||||
raise DataSourceAuthenticationError(
|
||||
"缺少 TUSHARE_TOKEN：请 cp .env.example .env 并填入 Tushare Pro token"
|
||||
)
|
||||
try:
|
||||
ts = importlib.import_module("tushare")
|
||||
except ImportError as exc: # pragma: no cover —— 环境相关
|
||||
raise DataSourceError(
|
||||
"未安装 tushare 客户端：cd backend && uv sync --extra datasource-tushare"
|
||||
) from exc
|
||||
return ts.pro_api(token)
|
||||
Reference in New Issue
Block a user