Simon
23972e7063
汇总三轮未提交的开发（每轮均在本机 MariaDB + 真实浏览器上验证）：
1) 股息率案例（全市场股息率最高 n 只，默认 20，每 m 月择股）
- 新增日频估值表 daily_basic + 迁移；股息率因子（dv_ratio / dividend_yield / TTM）
- 名称历史表 stock_name_history：剔除 ST 按**择股日当时名称**判定，消除
「曾高股息后 ST」的股息陷阱（实测 3.70pp 偏差）
- 区间择股/调仓双周期（m 择股 / y 调仓）、指数成分与白名单、停牌近似剔除
- 复权因子口径核对（4,164,742 行、缺失 0.0%）、收盘价成交与涨跌停拦单
- 案例实测：2020-01-01~2026-09-04 总收益 +24.86%（年化 3.52%、回撤 -28.58%）
2) 策略库与前端统一
- strategy 表 + CRUD/PUT 原地更新 + `describe_strategy` 按 spec 真实推导
「一句话说明 + 计算公式 + 执行步骤 + 注意事项」（与引擎实执行规则同源）
- 任何出现股票代码处都成对显示名称且可点击进个股页
- 全站图表基座统一 TradingView Lightweight Charts（ECharts 依赖、
锁文件、组件与文档标注一并清除），买卖点标记只落在真实交易日上
3) 回测存档完整化（可往复查看）
- 同步端点（POST /api/backtests、/api/factor-tests）此前完全不落库 → 现在同样归档，
归档 id 经响应头 X-Experiment-Id 返回（不破坏 response_model）
- data_version 首次真实写入（数据快照指纹：最新交易日 + 各表规模）
- 个股收益曲线默认**全量保存**（此前硬截断 60 只）；超出体积预算才裁剪，
并写 archive_meta（机器可读）+ unimplemented（人可读）如实标注
- 列表 kind/q 过滤 + X-Total-Count（此前 limit=50 静默截断）、DELETE 归档
- 只读归档页 /experiments/{id}（Server Component，SSR 直出**选股条件**与
**交易执行依据**）；结果视图按 kind 分发（backtest/factor_test/selection），
非回测归档不套用回测口径
- 新增 CLI：prune_experiments（保留策略，默认 dry-run）、
restore_experiment_from_job（从 Job 副本按原 id 重建被删的历史归档，默认 dry-run）
门禁：pytest 388 passed、ruff All checks passed、tsc 0 错误、图表单测 7 passed、
next build 成功、契约脚本 verify_strategy_workspace 59/59（含按 kind 逐类验证归档页）。
812 lines
30 KiB
Python
812 lines
30 KiB
Python
"""数据同步服务：增量 + 新浪「两边一致」校验兜底（financial / daily）。
|
||
|
||
背景（AGENT.md §5/§7/§8）：
|
||
- Tushare 是首选源；新浪财经只作备用。任何切源都必须可追溯（写 sync_log），
|
||
且禁止静默把未经核验的备用源数据并入主库。
|
||
- 本模块把「切到新浪」从 FailoverProvider 的『主源报错即兜底』收紧为
|
||
『校验兜底』：只有当某只股票**两边重叠的历史数据一致**时，才允许把新浪
|
||
的**新数据**（本地缺失键的行）导入；无本地历史或校验不一致 → 拒绝并告警，
|
||
留待 Tushare 恢复后重跑补齐（数据真实性优先）。
|
||
|
||
校验口径（经验证，见仓库数据）：
|
||
- 财务可比字段只有 eps / gross_margin —— 两源同报告期数值逐位一致；
|
||
ROE 两边口径不同（Tushare 摊薄 vs 新浪加权），不作为一致性依据。
|
||
- 日线新浪为前复权，与本地不复权行仅「最近无除权区间」相等，因此只拿
|
||
两源重叠的最近若干个交易日做一致性校验（通道可信 → 才允许补缺）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from collections.abc import Callable, Sequence
|
||
from dataclasses import dataclass, field
|
||
from datetime import date, timedelta
|
||
from decimal import Decimal
|
||
|
||
from app.domain.entities.market import DailyBar, FinancialIndicator, SyncLog
|
||
from app.domain.providers import MarketDataProvider
|
||
from app.domain.repositories.market import (
|
||
AdjustFactorRepository,
|
||
DailyBarRepository,
|
||
FinancialRepository,
|
||
)
|
||
from app.infrastructure.data_sources.errors import DataSourceAuthenticationError
|
||
|
||
# 财务两源可比字段（其余字段两端口径不一致 / 单侧缺失，不能作校验依据）
|
||
FINANCIAL_COMPARE_FIELDS = ("eps", "gross_margin")
|
||
DAILY_COMPARE_FIELDS = ("open", "high", "low", "close")
|
||
|
||
# 新浪日 K 可达窗口（getKLineData datalen=320 自然日）
|
||
SINA_KLINE_DAYS = 320
|
||
# 校验回看：请求新浪时额外回看 begin 之前的天数，确保与本地近期历史有重叠可比
|
||
SINA_VERIFY_LOOKBACK_DAYS = 45
|
||
_EPOCH = date(1990, 1, 1)
|
||
|
||
|
||
# ---------------------------------------------------------------- 一致性校验（纯函数）
|
||
|
||
@dataclass
|
||
class OverlapVerdict:
|
||
"""两边重叠一致性结论。ok=True 才允许导入新浪新数据。"""
|
||
|
||
ok: bool
|
||
shared: int = 0 # 重叠报告期 / 重叠交易日数量
|
||
compared: int = 0 # 实际参与数值比较的行/日数量
|
||
mismatches: list[str] = field(default_factory=list)
|
||
|
||
def summary(self) -> str:
|
||
if self.ok:
|
||
return f"重叠 {self.shared} 项，数值一致（比较 {self.compared} 项）"
|
||
why = f"重叠 {self.shared} 项不足/为空"
|
||
if self.mismatches:
|
||
why = "；".join(self.mismatches[:3])
|
||
return f"校验未通过：{why}"
|
||
|
||
|
||
def _close_enough(a: Decimal, b: Decimal, *, rel_tol: float, abs_tol: float) -> bool:
|
||
if a is None or b is None:
|
||
return False
|
||
diff = abs(a - b)
|
||
if diff <= Decimal(str(abs_tol)):
|
||
return True
|
||
scale = max(abs(a), abs(b))
|
||
return diff <= Decimal(str(rel_tol)) * scale
|
||
|
||
|
||
def financial_overlap_consistent(
|
||
local_rows: Sequence[FinancialIndicator],
|
||
sina_rows: Sequence[FinancialIndicator],
|
||
*,
|
||
min_shared: int = 2,
|
||
rel_tol: float = 1e-4,
|
||
abs_tol: float = 1e-3,
|
||
) -> OverlapVerdict:
|
||
"""新浪财务行与本地（Tushare）行按报告期重叠校验。
|
||
|
||
新浪每个报告期只保留最新一版（getFinanceReport2022 的 report_list 按
|
||
报告期一份）；本地同报告期可能有多版公告，取公告日最新者比较。
|
||
要求：重叠报告期数 >= min_shared，且全部可比字段（两源都非空）一致。
|
||
"""
|
||
local_latest: dict[date, FinancialIndicator] = {}
|
||
for row in local_rows:
|
||
cur = local_latest.get(row.report_date)
|
||
if cur is None or row.announce_date > cur.announce_date:
|
||
local_latest[row.report_date] = row
|
||
sina_by_report = {row.report_date: row for row in sina_rows}
|
||
|
||
verdict = OverlapVerdict(ok=False)
|
||
shared_dates = sorted(set(local_latest) & set(sina_by_report), reverse=True)
|
||
verdict.shared = len(shared_dates)
|
||
for report in shared_dates:
|
||
a = local_latest[report]
|
||
b = sina_by_report[report]
|
||
day_mismatch: list[str] = []
|
||
compared = 0
|
||
for f in FINANCIAL_COMPARE_FIELDS:
|
||
va, vb = getattr(a, f), getattr(b, f)
|
||
if va is None or vb is None:
|
||
continue
|
||
compared += 1
|
||
if not _close_enough(va, vb, rel_tol=rel_tol, abs_tol=abs_tol):
|
||
day_mismatch.append(f"{report}: {f} {va}≠{vb}")
|
||
verdict.compared += compared
|
||
verdict.mismatches.extend(day_mismatch)
|
||
verdict.ok = (
|
||
verdict.shared >= min_shared and verdict.compared > 0 and not verdict.mismatches
|
||
)
|
||
return verdict
|
||
|
||
|
||
def daily_overlap_consistent(
|
||
local_bars: Sequence[DailyBar],
|
||
sina_bars: Sequence[DailyBar],
|
||
*,
|
||
min_shared: int = 3,
|
||
max_recent: int = 8,
|
||
rel_tol: float = 1e-4,
|
||
abs_tol: float = Decimal("0.02"),
|
||
) -> OverlapVerdict:
|
||
"""新浪日 K（前复权）与本地（不复权）重叠校验。
|
||
|
||
前复权锚定最新价：仅「最近一次除权之后」的交易日两源数值相等，因此只
|
||
比较两源重叠的、最近的 max_recent 个交易日（此时若有除权发生在该段，
|
||
校验会判不一致 → 拒绝兜底，安全方向）。vol/amount 两源单位/口径不同，
|
||
不参与比较。
|
||
"""
|
||
local_by_day = {b.trade_date: b for b in local_bars}
|
||
sina_by_day = {b.trade_date: b for b in sina_bars}
|
||
shared = sorted(set(local_by_day) & set(sina_by_day), reverse=True)
|
||
|
||
verdict = OverlapVerdict(ok=False)
|
||
verdict.shared = len(shared)
|
||
for day in shared[:max_recent]:
|
||
a, b = local_by_day[day], sina_by_day[day]
|
||
day_mismatch: list[str] = []
|
||
compared = 0
|
||
for f in DAILY_COMPARE_FIELDS:
|
||
va, vb = getattr(a, f), getattr(b, f)
|
||
if va is None or vb is None:
|
||
continue
|
||
compared += 1
|
||
if not _close_enough(va, vb, rel_tol=rel_tol, abs_tol=abs_tol):
|
||
day_mismatch.append(f"{day}: {f} {va}≠{vb}")
|
||
verdict.compared += compared
|
||
verdict.mismatches.extend(day_mismatch)
|
||
checked = len(shared[:max_recent])
|
||
verdict.ok = (
|
||
checked >= min_shared and verdict.compared > 0 and not verdict.mismatches
|
||
)
|
||
return verdict
|
||
|
||
|
||
# ---------------------------------------------------------------- 报告期披露节奏
|
||
|
||
def latest_expected_report_period(today: date) -> date:
|
||
"""当前「应已披露」的最新报告期（报告期结束日）。
|
||
|
||
用作财务增量的已最新判断：本地已含该报告期 → 该股票已跟进到最新一季，
|
||
跳过（避免每轮全量重拉；--full 强制）。窗口按 A 股披露节奏划分：
|
||
- 1/1~2/14：年报季未开 → 上年三季报(09-30)
|
||
- 2/15~6/30：年报+一季报季 → 本年一季报(03-31)
|
||
- 7/1~10/15：半年报季 → 本年半年报(06-30)
|
||
- 10/16~12/31：三季报季 → 本年三季报(09-30)
|
||
"""
|
||
y = today.year
|
||
md = (today.month, today.day)
|
||
if md <= (2, 14):
|
||
return date(y - 1, 9, 30)
|
||
if md <= (6, 30):
|
||
return date(y, 3, 31)
|
||
if md <= (10, 15):
|
||
return date(y, 6, 30)
|
||
return date(y, 9, 30)
|
||
|
||
|
||
# ---------------------------------------------------------------- 审计
|
||
|
||
def _audit_sync(
|
||
audit: Callable[[SyncLog], None],
|
||
*,
|
||
source: str,
|
||
api: str,
|
||
success: bool,
|
||
row_count: int = 0,
|
||
reason: str | None = None,
|
||
start: date | None = None,
|
||
end: date | None = None,
|
||
) -> None:
|
||
audit(
|
||
SyncLog(
|
||
source=source,
|
||
api=api,
|
||
success=success,
|
||
failure_reason=reason,
|
||
row_count=row_count,
|
||
data_start=start,
|
||
data_end=end,
|
||
)
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------- 结果结构
|
||
|
||
@dataclass
|
||
class FinancialSymbolResult:
|
||
"""单只股票财务同步结果（status: skip|ok|sina|failed）。"""
|
||
|
||
symbol: str
|
||
status: str
|
||
source: str | None = None # tushare | sina
|
||
fetched: int = 0 # 数据源返回行数
|
||
written: int = 0 # 实际落库行数（新增；--full 时含更新）
|
||
updated: int = 0 # --full 下覆盖的既有行数
|
||
report_first: date | None = None
|
||
report_last: date | None = None
|
||
announce_first: date | None = None
|
||
announce_last: date | None = None
|
||
notes: list[str] = field(default_factory=list)
|
||
|
||
|
||
@dataclass
|
||
class DailySymbolResult:
|
||
"""单只股票日线同步结果（status: skip|ok|sina|failed）。"""
|
||
|
||
symbol: str
|
||
status: str
|
||
source: str | None = None # tushare | sina
|
||
bars_fetched: int = 0
|
||
bars_written: int = 0
|
||
day_first: date | None = None
|
||
day_last: date | None = None
|
||
factors_written: int | None = None # None=未尝试（新浪兜底无因子）
|
||
notes: list[str] = field(default_factory=list)
|
||
|
||
|
||
@dataclass
|
||
class DailyBasicDayResult:
|
||
"""单个交易日的每日指标同步结果（status: skip|ok|failed）。"""
|
||
|
||
trade_date: date
|
||
status: str
|
||
source: str | None = None # tushare（新浪不支持本接口）
|
||
rows_fetched: int = 0
|
||
rows_written: int = 0
|
||
notes: list[str] = field(default_factory=list)
|
||
|
||
|
||
@dataclass
|
||
class NameHistoryChunkResult:
|
||
"""单个时间分片的名称变更同步结果（status: ok|failed）。"""
|
||
|
||
start: date
|
||
end: date
|
||
status: str
|
||
source: str | None = None
|
||
rows_fetched: int = 0
|
||
rows_written: int = 0
|
||
notes: list[str] = field(default_factory=list)
|
||
|
||
|
||
# ---------------------------------------------------------------- 财务同步服务
|
||
|
||
class VerifiedFinancialSyncer:
|
||
"""财务指标增量同步：Tushare 窗口化拉取 → 失败则新浪校验兜底。"""
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
primary: MarketDataProvider,
|
||
fallback: MarketDataProvider | None,
|
||
repo: FinancialRepository,
|
||
audit: Callable[[SyncLog], None],
|
||
today: date | None = None,
|
||
min_shared: int = 2,
|
||
) -> None:
|
||
self.primary = primary
|
||
self.fallback = fallback
|
||
self.repo = repo
|
||
self.audit = audit
|
||
self.today = today or date.today()
|
||
self.min_shared = min_shared
|
||
|
||
def sync_symbol(self, symbol: str, *, force_full: bool = False) -> FinancialSymbolResult:
|
||
local = self.repo.list_symbol(symbol)
|
||
local_keys = {(r.symbol, r.report_date, r.announce_date) for r in local}
|
||
due = latest_expected_report_period(self.today)
|
||
if not force_full and local and any(r.report_date == due for r in local):
|
||
return FinancialSymbolResult(
|
||
symbol=symbol,
|
||
status="skip",
|
||
notes=[f"本地已含最新报告期 {due.isoformat()}，跳过（--full 强制重拉）"],
|
||
)
|
||
# 拉取窗口：有本地行则从最早本地报告期起（含更正/补缺），无则全历史；
|
||
# 上限到最新应披露报告期。
|
||
hi = due
|
||
lo = min((r.report_date for r in local), default=None) or _EPOCH
|
||
try:
|
||
rows = self.primary.get_financial(symbol, lo, hi)
|
||
except DataSourceAuthenticationError:
|
||
# 凭证无效/接口无权限：属全局性故障，快速失败让用户修 token，
|
||
# 不要对全市场逐只做无意义的新浪试探
|
||
raise
|
||
except Exception as exc: # noqa: BLE001 —— 与 FailoverProvider 一致，统一走审计
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_financial",
|
||
success=False,
|
||
reason=str(exc),
|
||
start=lo,
|
||
end=hi,
|
||
)
|
||
return self._sina_fallback(symbol, local, local_keys, primary_error=str(exc))
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_financial",
|
||
success=True,
|
||
row_count=len(rows),
|
||
start=lo,
|
||
end=hi,
|
||
)
|
||
if force_full:
|
||
to_write = rows
|
||
updated = sum(1 for r in rows if _fin_key(r) in local_keys)
|
||
else:
|
||
to_write = [r for r in rows if _fin_key(r) not in local_keys]
|
||
updated = 0
|
||
written = self.repo.upsert_many(to_write)
|
||
return _fin_result(symbol, status="ok", source="tushare", written_rows=to_write,
|
||
written=written, updated=updated)
|
||
|
||
# ---- 新浪校验兜底 ----
|
||
|
||
def _sina_fallback(
|
||
self,
|
||
symbol: str,
|
||
local: list[FinancialIndicator],
|
||
local_keys: set[tuple],
|
||
*,
|
||
primary_error: str,
|
||
) -> FinancialSymbolResult:
|
||
if self.fallback is None:
|
||
return FinancialSymbolResult(
|
||
symbol=symbol,
|
||
status="failed",
|
||
notes=[f"Tushare 失败且未配置新浪兜底: {primary_error}"],
|
||
)
|
||
if not local:
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.fallback.name,
|
||
api="get_financial",
|
||
success=False,
|
||
reason=f"无本地历史可做两边一致性校验，跳过待 Tushare 恢复重试（{primary_error}）",
|
||
)
|
||
return FinancialSymbolResult(
|
||
symbol=symbol,
|
||
status="failed",
|
||
source="sina",
|
||
notes=[
|
||
f"Tushare 失败且本地无历史（{symbol}），无法确认新浪数据真实性，"
|
||
f"跳过待重试。primary: {primary_error}"
|
||
],
|
||
)
|
||
try:
|
||
sina_rows = self.fallback.get_financial(symbol)
|
||
except Exception as exc: # noqa: BLE001
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.fallback.name,
|
||
api="get_financial",
|
||
success=False,
|
||
reason=f"primary: {primary_error}; fallback: {exc}",
|
||
)
|
||
return FinancialSymbolResult(
|
||
symbol=symbol,
|
||
status="failed",
|
||
source="sina",
|
||
notes=[f"主备数据源均失败: primary={primary_error}; sina={exc}"],
|
||
)
|
||
verdict = financial_overlap_consistent(local, sina_rows, min_shared=self.min_shared)
|
||
if not verdict.ok:
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.fallback.name,
|
||
api="get_financial",
|
||
success=False,
|
||
reason=f"{verdict.summary()}（新浪返回 {len(sina_rows)} 行; primary={primary_error}）",
|
||
)
|
||
return FinancialSymbolResult(
|
||
symbol=symbol,
|
||
status="failed",
|
||
source="sina",
|
||
notes=[
|
||
f"新浪数据与本地历史不一致/无法校验（{symbol}），拒绝导入。"
|
||
f"primary: {primary_error}；{verdict.summary()}"
|
||
],
|
||
)
|
||
new_rows = [r for r in sina_rows if _fin_key(r) not in local_keys]
|
||
written = self.repo.upsert_many(new_rows)
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.fallback.name,
|
||
api="get_financial",
|
||
success=True,
|
||
row_count=written,
|
||
)
|
||
return _fin_result(symbol, status="sina", source="sina", written_rows=new_rows,
|
||
written=written, updated=0,
|
||
note=f"新浪校验通过后补入 {written} 行（仅本地缺失键，source=sina）")
|
||
|
||
|
||
# ---------------------------------------------------------------- 日线同步服务
|
||
|
||
class VerifiedDailySyncer:
|
||
"""日线同步：Tushare 失败 → 新浪校验兜底（仅补缺失交易日、无复权因子）。"""
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
primary: MarketDataProvider,
|
||
fallback: MarketDataProvider | None,
|
||
bars: DailyBarRepository,
|
||
factors: AdjustFactorRepository,
|
||
audit: Callable[[SyncLog], None],
|
||
today: date | None = None,
|
||
min_shared: int = 3,
|
||
) -> None:
|
||
self.primary = primary
|
||
self.fallback = fallback
|
||
self.bars = bars
|
||
self.factors = factors
|
||
self.audit = audit
|
||
self.today = today or date.today()
|
||
self.min_shared = min_shared
|
||
|
||
def sync_symbol(self, symbol: str, begin: date, end: date) -> DailySymbolResult:
|
||
try:
|
||
bars = self.primary.get_daily(symbol, begin, end)
|
||
except DataSourceAuthenticationError:
|
||
raise # 凭证/权限故障 → 快速失败（见财务同步注释）
|
||
except Exception as exc: # noqa: BLE001
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_daily",
|
||
success=False,
|
||
reason=str(exc),
|
||
start=begin,
|
||
end=end,
|
||
)
|
||
return self._sina_fallback(symbol, begin, end, primary_error=str(exc))
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_daily",
|
||
success=True,
|
||
row_count=len(bars),
|
||
start=begin,
|
||
end=end,
|
||
)
|
||
try:
|
||
factors = self.primary.get_adjust_factor(symbol, begin, end)
|
||
except DataSourceAuthenticationError:
|
||
raise # 凭证/权限故障 → 快速失败（见财务同步注释）
|
||
except Exception as exc: # noqa: BLE001
|
||
# 复权因子是日线配套：缺因子不写本段，避免 resume 按日线已最新而跳过、因子永远补不上
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_adjust_factor",
|
||
success=False,
|
||
reason=str(exc),
|
||
start=begin,
|
||
end=end,
|
||
)
|
||
return DailySymbolResult(
|
||
symbol=symbol,
|
||
status="failed",
|
||
source="tushare",
|
||
bars_fetched=len(bars),
|
||
notes=[f"日线拉取成功但复权因子失败，本段未落库（防因子缺口）: {exc}"],
|
||
)
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_adjust_factor",
|
||
success=True,
|
||
row_count=len(factors),
|
||
start=begin,
|
||
end=end,
|
||
)
|
||
self.bars.upsert_many(bars)
|
||
self.factors.upsert_many(factors)
|
||
return DailySymbolResult(
|
||
symbol=symbol,
|
||
status="ok",
|
||
source="tushare",
|
||
bars_fetched=len(bars),
|
||
bars_written=len(bars),
|
||
factors_written=len(factors),
|
||
day_first=min((b.trade_date for b in bars), default=None),
|
||
day_last=max((b.trade_date for b in bars), default=None),
|
||
)
|
||
|
||
# ---- 新浪校验兜底 ----
|
||
|
||
def _sina_fallback(
|
||
self, symbol: str, begin: date, end: date, *, primary_error: str
|
||
) -> DailySymbolResult:
|
||
if self.fallback is None:
|
||
return DailySymbolResult(
|
||
symbol=symbol,
|
||
status="failed",
|
||
notes=[f"Tushare 失败且未配置新浪兜底: {primary_error}"],
|
||
)
|
||
# 新浪只有最近 SINA_KLINE_DAYS 自然日数据；为拿到「本地近期历史」重叠做
|
||
# 校验，请求窗口需回看 begin 之前 SINA_VERIFY_LOOKBACK_DAYS 天
|
||
# （见 daily_overlap_consistent：只比较两源重叠的最近交易日）。
|
||
q_start = max(
|
||
self.today - timedelta(days=SINA_KLINE_DAYS - 1),
|
||
begin - timedelta(days=SINA_VERIFY_LOOKBACK_DAYS),
|
||
)
|
||
try:
|
||
sina_bars = self.fallback.get_daily(symbol, q_start, end)
|
||
except Exception as exc: # noqa: BLE001
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.fallback.name,
|
||
api="get_daily",
|
||
success=False,
|
||
reason=f"primary: {primary_error}; fallback: {exc}",
|
||
start=q_start,
|
||
end=end,
|
||
)
|
||
return DailySymbolResult(
|
||
symbol=symbol,
|
||
status="failed",
|
||
source="sina",
|
||
notes=[f"主备数据源均失败: primary={primary_error}; sina={exc}"],
|
||
)
|
||
local_recent = self.bars.get_range(symbol, q_start, end)
|
||
verdict = daily_overlap_consistent(local_recent, sina_bars, min_shared=self.min_shared)
|
||
if not verdict.ok:
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.fallback.name,
|
||
api="get_daily",
|
||
success=False,
|
||
reason=f"{verdict.summary()}（新浪返回 {len(sina_bars)} 行; primary={primary_error}）",
|
||
start=q_start,
|
||
end=end,
|
||
)
|
||
return DailySymbolResult(
|
||
symbol=symbol,
|
||
status="failed",
|
||
source="sina",
|
||
bars_fetched=len(sina_bars),
|
||
notes=[
|
||
f"新浪数据与本地历史不一致/无法校验（{symbol}），拒绝兜底补缺。"
|
||
f"primary: {primary_error}；{verdict.summary()}"
|
||
],
|
||
)
|
||
local_dates = {b.trade_date for b in local_recent}
|
||
new_bars = [
|
||
b
|
||
for b in sina_bars
|
||
if begin <= b.trade_date <= end and b.trade_date not in local_dates
|
||
]
|
||
written = self.bars.upsert_many(new_bars)
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.fallback.name,
|
||
api="get_daily",
|
||
success=True,
|
||
row_count=written,
|
||
start=begin,
|
||
end=end,
|
||
)
|
||
return DailySymbolResult(
|
||
symbol=symbol,
|
||
status="sina",
|
||
source="sina",
|
||
bars_fetched=len(sina_bars),
|
||
bars_written=written,
|
||
day_first=min((b.trade_date for b in new_bars), default=None),
|
||
day_last=max((b.trade_date for b in new_bars), default=None),
|
||
factors_written=None,
|
||
notes=[
|
||
f"新浪校验通过，仅补本地缺失交易日 {written} 根（前复权 source=sina，"
|
||
f"无复权因子；Tushare 恢复后 --resume 会按日覆盖回不复权口径）"
|
||
],
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------- 每日指标同步服务
|
||
|
||
class DailyBasicSyncer:
|
||
"""每日指标（估值 / 股息率 / 市值）同步：按交易日整表拉取。
|
||
|
||
设计（与财务/日线的「逐股校验兜底」不同）：
|
||
- daily_basic 是**横截面整表**接口，Tushare 一次返回当日全市场，无法逐股兜底；
|
||
- 新浪不提供本接口 → 主源失败时**如实失败并写 sync_log**，禁止静默留缺口；
|
||
- 幂等键 (symbol, trade_date)：重跑同日无副作用（upsert）。
|
||
- 交易日集合由日历仓储给出：只同步 is_open 且本地缺失的日期
|
||
（`missing_dates`），因此增量与断点续跑天然安全。
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
primary: MarketDataProvider,
|
||
repo,
|
||
audit: Callable[[SyncLog], None],
|
||
batch_flush: int = 5,
|
||
) -> None:
|
||
self.primary = primary
|
||
self.repo = repo
|
||
self.audit = audit
|
||
self.batch_flush = max(batch_flush, 1)
|
||
|
||
def sync_day(self, trade_date: date) -> DailyBasicDayResult:
|
||
try:
|
||
rows = self.primary.get_daily_basic(trade_date)
|
||
except DataSourceAuthenticationError:
|
||
raise # 凭证/权限故障 → 快速失败（不要逐日重试把配额烧光）
|
||
except Exception as exc: # noqa: BLE001 —— 逐日失败不中断整段
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_daily_basic",
|
||
success=False,
|
||
reason=str(exc),
|
||
start=trade_date,
|
||
end=trade_date,
|
||
)
|
||
return DailyBasicDayResult(
|
||
trade_date=trade_date,
|
||
status="failed",
|
||
source=self.primary.name,
|
||
notes=[f"拉取失败: {exc}"],
|
||
)
|
||
# 过滤掉非法行（ts_code 缺失）—— 避免脏键污染幂等
|
||
rows = [r for r in rows if r.symbol and r.symbol != "None"]
|
||
written = self.repo.upsert_many(rows)
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_daily_basic",
|
||
success=True,
|
||
row_count=len(rows),
|
||
start=trade_date,
|
||
end=trade_date,
|
||
)
|
||
return DailyBasicDayResult(
|
||
trade_date=trade_date,
|
||
status="ok",
|
||
source=self.primary.name,
|
||
rows_fetched=len(rows),
|
||
rows_written=written,
|
||
)
|
||
|
||
def sync_range(
|
||
self,
|
||
start: date,
|
||
end: date,
|
||
*,
|
||
on_progress: Callable[[int, int, date], None] | None = None,
|
||
should_stop: Callable[[], bool] | None = None,
|
||
) -> list[DailyBasicDayResult]:
|
||
"""补齐 [start, end] 内开市但本地缺失的交易日。
|
||
|
||
on_progress(done, total, trade_date)：进度回调（CLI 打印 / 日志）。
|
||
should_stop()：返回 True 时提前收尾（优雅中断，已落库的行保持有效）。
|
||
"""
|
||
days = self.repo.missing_dates(start, end)
|
||
results: list[DailyBasicDayResult] = []
|
||
total = len(days)
|
||
for idx, day in enumerate(days, start=1):
|
||
if should_stop is not None and should_stop():
|
||
break
|
||
results.append(self.sync_day(day))
|
||
if on_progress is not None:
|
||
on_progress(idx, total, day)
|
||
return results
|
||
|
||
|
||
class NameHistorySyncer:
|
||
"""股票名称变更历史同步（Tushare namechange）—— 时点 ST 判定的数据基础。
|
||
|
||
设计要点：
|
||
- **按年分片**：namechange 支持区间批量查询（2020+ 仅 4031 行），但全历史
|
||
（1990 起）会触及单次 6000 行上限被**静默截断**，因此按自然年分片调用，
|
||
每片独立审计与计数，超限时 Provider 会告警。
|
||
- **幂等键 (symbol, start_date)**：重跑无副作用（upsert），可安全续跑。
|
||
- 新浪不提供本接口 → 主源失败如实写 sync_log（禁止静默留缺口，AGENT.md §24）。
|
||
- 为什么值得同步：`stock.name` 只是最新名称快照，用最新名称做 `exclude_st`
|
||
会把「曾为高股息、后变 ST/退市」的股息陷阱样本整段排除 ——
|
||
实测影响约 3.70pp 收益（见 docs/DEV_PLAN_DIVIDEND_BACKTEST.md §10.5）。
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
primary: MarketDataProvider,
|
||
repo,
|
||
audit: Callable[[SyncLog], None],
|
||
) -> None:
|
||
self.primary = primary
|
||
self.repo = repo
|
||
self.audit = audit
|
||
|
||
def sync_chunk(self, start: date, end: date) -> NameHistoryChunkResult:
|
||
try:
|
||
rows = self.primary.get_name_changes(start, end)
|
||
except DataSourceAuthenticationError:
|
||
raise # 凭证/权限故障 → 快速失败，不要逐片重试烧配额
|
||
except Exception as exc: # noqa: BLE001 —— 单片失败不中断整段
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_namechange",
|
||
success=False,
|
||
reason=str(exc),
|
||
start=start,
|
||
end=end,
|
||
)
|
||
return NameHistoryChunkResult(
|
||
start=start, end=end, status="failed",
|
||
source=self.primary.name, notes=[f"拉取失败: {exc}"],
|
||
)
|
||
rows = [r for r in rows if r.symbol and r.symbol != "None"]
|
||
written = self.repo.upsert_many(rows)
|
||
_audit_sync(
|
||
self.audit,
|
||
source=self.primary.name,
|
||
api="get_namechange",
|
||
success=True,
|
||
row_count=len(rows),
|
||
start=start,
|
||
end=end,
|
||
)
|
||
return NameHistoryChunkResult(
|
||
start=start, end=end, status="ok", source=self.primary.name,
|
||
rows_fetched=len(rows), rows_written=written,
|
||
)
|
||
|
||
def sync_range(
|
||
self,
|
||
start: date,
|
||
end: date,
|
||
*,
|
||
on_progress: Callable[[int, int, date], None] | None = None,
|
||
) -> list[NameHistoryChunkResult]:
|
||
"""按自然年分片同步 [start, end]（每片 1 次 API 调用）。"""
|
||
chunks: list[tuple[date, date]] = []
|
||
cursor = date(start.year, 1, 1)
|
||
while cursor <= end:
|
||
chunk_end = min(date(cursor.year, 12, 31), end)
|
||
chunks.append((max(cursor, start), chunk_end))
|
||
cursor = date(cursor.year + 1, 1, 1)
|
||
results: list[NameHistoryChunkResult] = []
|
||
total = len(chunks)
|
||
for idx, (cs, ce) in enumerate(chunks, start=1):
|
||
results.append(self.sync_chunk(cs, ce))
|
||
if on_progress is not None:
|
||
on_progress(idx, total, cs)
|
||
return results
|
||
|
||
|
||
# ---------------------------------------------------------------- 小工具
|
||
|
||
def _fin_key(row: FinancialIndicator) -> tuple:
|
||
return (row.symbol, row.report_date, row.announce_date)
|
||
|
||
|
||
def _fin_result(
|
||
symbol: str,
|
||
*,
|
||
status: str,
|
||
source: str,
|
||
written_rows: Sequence[FinancialIndicator],
|
||
written: int,
|
||
updated: int,
|
||
note: str | None = None,
|
||
) -> FinancialSymbolResult:
|
||
reports = [r.report_date for r in written_rows]
|
||
announces = [r.announce_date for r in written_rows]
|
||
notes = [note] if note else []
|
||
return FinancialSymbolResult(
|
||
symbol=symbol,
|
||
status=status,
|
||
source=source,
|
||
fetched=len(written_rows),
|
||
written=written,
|
||
updated=updated,
|
||
report_first=min(reports, default=None),
|
||
report_last=max(reports, default=None),
|
||
announce_first=min(announces, default=None),
|
||
announce_last=max(announces, default=None),
|
||
notes=notes,
|
||
)
|