Simon
442999f701
- financial 默认增量：按 A 股披露节奏判断已最新并跳过；--full 强制全量重拉 - Tushare fina_indicator 增加报告期窗口与 100 条/请求自动分页（修复老报告期静默截断） - 新浪兜底收紧为校验兜底：两源重叠历史一致才导入缺失键，行标记 source=sina； 财务可比字段取 eps/销售毛利率（ROE 两端口径不同不作依据），日线只比较最近重叠交易日 - CLI 输出逐只进度与导入内容描述（来源/行数/报告期与公告区间），失败股票留待重跑 - financial_indicator 增 source 列（迁移 d3f6c9a21b04）；新增一致性/分页/服务测试
451 lines
18 KiB
Python
451 lines
18 KiB
Python
"""增量 + 新浪「两边一致」校验兜底同步服务测试。
|
||
|
||
覆盖：一致性校验纯函数、报告期节奏、财务/日线同步服务
|
||
（temp SQLite + Fake Provider，不触网）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import date
|
||
from decimal import Decimal
|
||
|
||
import pytest
|
||
from app.application.services.data_sync import (
|
||
VerifiedDailySyncer,
|
||
VerifiedFinancialSyncer,
|
||
daily_overlap_consistent,
|
||
financial_overlap_consistent,
|
||
latest_expected_report_period,
|
||
)
|
||
from app.domain.entities.market import DailyBar, FinancialIndicator, SyncLog
|
||
from app.infrastructure.data_sources.errors import DataSourceError
|
||
from app.infrastructure.persistence.sqlalchemy.base import Base
|
||
from app.infrastructure.persistence.sqlalchemy.models.market import (
|
||
AdjustFactorModel,
|
||
FinancialIndicatorModel,
|
||
StockDailyModel,
|
||
)
|
||
from app.infrastructure.persistence.sqlalchemy.repositories.market_impl import (
|
||
SqlAlchemyAdjustFactorRepository,
|
||
SqlAlchemyDailyBarRepository,
|
||
SqlAlchemyFinancialRepository,
|
||
)
|
||
from sqlalchemy import create_engine, func, select
|
||
from sqlalchemy.orm import Session
|
||
|
||
TODAY = date(2026, 9, 8)
|
||
DUE = date(2026, 6, 30) # TODAY 属于半年报季 → 应披露最新报告期
|
||
|
||
|
||
@pytest.fixture()
|
||
def session(tmp_path) -> Session:
|
||
engine = create_engine(f"sqlite:///{tmp_path / 'ds.db'}", future=True)
|
||
Base.metadata.create_all(engine)
|
||
with Session(engine) as session:
|
||
yield session
|
||
|
||
|
||
def _fin(
|
||
symbol: str,
|
||
report: str,
|
||
announce: str,
|
||
*,
|
||
eps: str = "1.00",
|
||
roe: str | None = None,
|
||
gross: str | None = None,
|
||
source: str = "tushare",
|
||
) -> FinancialIndicator:
|
||
return FinancialIndicator(
|
||
symbol=symbol,
|
||
report_date=date.fromisoformat(report),
|
||
announce_date=date.fromisoformat(announce),
|
||
source=source,
|
||
eps=Decimal(eps),
|
||
roe=Decimal(roe) if roe is not None else None,
|
||
gross_margin=Decimal(gross) if gross is not None else None,
|
||
)
|
||
|
||
|
||
def _bar(symbol: str, day: str, close: str, source: str = "tushare") -> DailyBar:
|
||
return DailyBar(
|
||
symbol=symbol,
|
||
trade_date=date.fromisoformat(day),
|
||
source=source,
|
||
adjust="none" if source == "tushare" else "qfq",
|
||
open=Decimal(close),
|
||
high=Decimal(close),
|
||
low=Decimal(close),
|
||
close=Decimal(close),
|
||
volume=Decimal("1000"),
|
||
)
|
||
|
||
|
||
# ================================================================ 一致性校验
|
||
|
||
class TestFinancialConsistency:
|
||
def test_overlap_consistent_passes(self) -> None:
|
||
local = [
|
||
_fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", roe="10.5687", gross="89.7592"),
|
||
_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", roe="17.9543", gross="89.5552"),
|
||
]
|
||
sina = [
|
||
_fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", roe="10.57", gross="89.759217", source="sina"),
|
||
_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", roe="16.75", gross="89.555212", source="sina"),
|
||
]
|
||
verdict = financial_overlap_consistent(local, sina)
|
||
# ROE 两边口径不同（新浪加权 vs tushare 摊薄），不参与比较也不判失败
|
||
assert verdict.ok
|
||
assert verdict.shared == 2
|
||
assert not verdict.mismatches
|
||
|
||
def test_roe_difference_not_compared(self) -> None:
|
||
"""口径差异字段绝不能导致误杀（17.9543 vs 16.75 是同源数据真实形态）。"""
|
||
local = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", roe="17.9543", gross="89.5552")]
|
||
sina = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", roe="16.75", gross="89.555212", source="sina")]
|
||
verdict = financial_overlap_consistent(local, sina, min_shared=1)
|
||
assert verdict.ok
|
||
|
||
def test_eps_mismatch_fails(self) -> None:
|
||
local = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", gross="89.5552")]
|
||
sina = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="99.99", gross="89.555212", source="sina")]
|
||
verdict = financial_overlap_consistent(local, sina, min_shared=1)
|
||
assert not verdict.ok
|
||
assert verdict.mismatches
|
||
|
||
def test_insufficient_overlap_fails(self) -> None:
|
||
local = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57")]
|
||
sina = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", source="sina")]
|
||
verdict = financial_overlap_consistent(local, sina, min_shared=2)
|
||
assert not verdict.ok
|
||
assert verdict.summary()
|
||
|
||
|
||
class TestDailyConsistency:
|
||
def test_recent_overlap_equal_passes(self) -> None:
|
||
local = [_bar("600519.SH", "2026-09-01", "10"), _bar("600519.SH", "2026-09-02", "10")]
|
||
sina = [
|
||
_bar("600519.SH", "2026-09-01", "10", source="sina"),
|
||
_bar("600519.SH", "2026-09-02", "10", source="sina"),
|
||
]
|
||
verdict = daily_overlap_consistent(local, sina, min_shared=2)
|
||
assert verdict.ok
|
||
|
||
def test_old_dividend_scale_gap_ignored(self) -> None:
|
||
"""新浪前复权：老交易日在除权后数值整体平移，只有「最近重叠段」可比。"""
|
||
local = [
|
||
_bar("600519.SH", "2025-06-02", "500"),
|
||
_bar("600519.SH", "2026-09-01", "10"),
|
||
_bar("600519.SH", "2026-09-02", "10"),
|
||
_bar("600519.SH", "2026-09-03", "10"),
|
||
]
|
||
sina = [
|
||
_bar("600519.SH", "2025-06-02", "480", source="sina"), # 除权后按因子平移
|
||
_bar("600519.SH", "2026-09-01", "10", source="sina"),
|
||
_bar("600519.SH", "2026-09-02", "10", source="sina"),
|
||
_bar("600519.SH", "2026-09-03", "10", source="sina"),
|
||
]
|
||
verdict = daily_overlap_consistent(local, sina, min_shared=2, max_recent=3)
|
||
assert verdict.ok
|
||
|
||
def test_recent_close_mismatch_fails(self) -> None:
|
||
local = [_bar("600519.SH", "2026-09-02", "10")]
|
||
sina = [_bar("600519.SH", "2026-09-02", "9", source="sina")]
|
||
verdict = daily_overlap_consistent(local, sina, min_shared=1)
|
||
assert not verdict.ok
|
||
|
||
|
||
class TestReportPeriodRhythm:
|
||
def test_windows(self) -> None:
|
||
assert latest_expected_report_period(date(2026, 1, 1)) == date(2025, 9, 30)
|
||
assert latest_expected_report_period(date(2026, 2, 14)) == date(2025, 9, 30)
|
||
assert latest_expected_report_period(date(2026, 2, 15)) == date(2026, 3, 31)
|
||
assert latest_expected_report_period(date(2026, 6, 30)) == date(2026, 3, 31)
|
||
assert latest_expected_report_period(date(2026, 7, 1)) == date(2026, 6, 30)
|
||
assert latest_expected_report_period(TODAY) == DUE
|
||
assert latest_expected_report_period(date(2026, 10, 15)) == date(2026, 6, 30)
|
||
assert latest_expected_report_period(date(2026, 10, 16)) == date(2026, 9, 30)
|
||
|
||
|
||
# ================================================================ 财务同步服务
|
||
|
||
class _FakeFinSource:
|
||
def __init__(self, *, name: str, payload=None, error: Exception | None = None) -> None:
|
||
self.name = name
|
||
self.payload = payload or []
|
||
self.error = error
|
||
self.calls: list[tuple] = []
|
||
|
||
def get_financial(self, symbol, start=None, end=None): # noqa: ANN001
|
||
self.calls.append((symbol, start, end))
|
||
if self.error is not None:
|
||
raise self.error
|
||
return list(self.payload)
|
||
|
||
|
||
def _fin_syncer(session, primary, fallback=None, *, today: date = TODAY):
|
||
logs: list[SyncLog] = []
|
||
syncer = VerifiedFinancialSyncer(
|
||
primary=primary,
|
||
fallback=fallback,
|
||
repo=SqlAlchemyFinancialRepository(session),
|
||
audit=logs.append,
|
||
today=today,
|
||
)
|
||
return syncer, logs
|
||
|
||
|
||
def _count(session, model) -> int:
|
||
return session.scalar(select(func.count()).select_from(model))
|
||
|
||
|
||
class TestVerifiedFinancialSyncer:
|
||
def _rows(self, symbol: str = "600519.SH") -> list[FinancialIndicator]:
|
||
return [
|
||
_fin(symbol, "2026-03-31", "2026-04-25", eps="21.76", gross="89.7592"),
|
||
_fin(symbol, "2026-06-30", "2026-08-15", eps="35.57", gross="89.5552"),
|
||
]
|
||
|
||
def test_first_fetch_writes_delta_and_second_run_skips(self, session: Session) -> None:
|
||
primary = _FakeFinSource(name="tushare", payload=self._rows())
|
||
syncer, logs = _fin_syncer(session, primary)
|
||
|
||
res = syncer.sync_symbol("600519.SH")
|
||
session.commit()
|
||
assert res.status == "ok"
|
||
assert res.written == 2
|
||
assert _count(session, FinancialIndicatorModel) == 2
|
||
assert logs[0].source == "tushare" and logs[0].success
|
||
|
||
# 第二次：本地已含最新应披露报告期 2026-06-30 → 跳过，不再调 API
|
||
res2 = syncer.sync_symbol("600519.SH")
|
||
assert res2.status == "skip"
|
||
assert len(primary.calls) == 1
|
||
|
||
def test_force_full_overwrites(self, session: Session) -> None:
|
||
primary = _FakeFinSource(name="tushare", payload=self._rows())
|
||
syncer, _logs = _fin_syncer(session, primary)
|
||
syncer.sync_symbol("600519.SH")
|
||
session.commit()
|
||
|
||
changed = [
|
||
r.model_copy(update={"eps": Decimal("99")}) for r in self._rows()
|
||
] + [_fin("600519.SH", "2025-12-31", "2026-04-17", eps="65.66", gross="91.1796")]
|
||
primary.payload = changed
|
||
res = syncer.sync_symbol("600519.SH", force_full=True)
|
||
session.commit()
|
||
assert res.status == "ok"
|
||
assert res.written == 3
|
||
assert res.updated == 2 # 既有 2 行被覆盖
|
||
got = session.scalars(select(FinancialIndicatorModel)).all()
|
||
assert len(got) == 3
|
||
assert max(float(r.eps) for r in got) == 99.0
|
||
|
||
def test_sina_fallback_verified_imports_only_new_keys(self, session: Session) -> None:
|
||
repo = SqlAlchemyFinancialRepository(session)
|
||
repo.upsert_many(
|
||
[
|
||
_fin("600519.SH", "2025-12-31", "2026-04-17", eps="65.66", gross="91.1796"),
|
||
_fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", gross="89.7592"),
|
||
]
|
||
)
|
||
session.commit()
|
||
|
||
sina_rows = [
|
||
_fin("600519.SH", "2025-12-31", "2026-04-17", eps="65.66", gross="91.179551", source="sina"),
|
||
_fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", gross="89.759217", source="sina"),
|
||
_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", gross="89.555212", source="sina"),
|
||
]
|
||
primary = _FakeFinSource(name="tushare", error=DataSourceError("频率超限"))
|
||
fallback = _FakeFinSource(name="sina", payload=sina_rows)
|
||
syncer, logs = _fin_syncer(session, primary, fallback)
|
||
|
||
res = syncer.sync_symbol("600519.SH")
|
||
session.commit()
|
||
assert res.status == "sina"
|
||
assert res.written == 1 # 只补本地缺失键（2026-06-30）
|
||
rows = session.scalars(select(FinancialIndicatorModel)).all()
|
||
assert len(rows) == 3
|
||
new = next(r for r in rows if r.report_date == DUE)
|
||
assert new.source == "sina"
|
||
# 审计：tushare 失败 + sina 成功
|
||
assert [log.source for log in logs] == ["tushare", "sina"]
|
||
assert logs[0].success is False and logs[1].success is True
|
||
|
||
def test_sina_fallback_denied_on_mismatch(self, session: Session) -> None:
|
||
repo = SqlAlchemyFinancialRepository(session)
|
||
# 用非「最新应披露期」做种子，避免触发「已最新跳过」
|
||
repo.upsert_many(
|
||
[_fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", gross="89.7592")]
|
||
)
|
||
session.commit()
|
||
|
||
primary = _FakeFinSource(name="tushare", error=DataSourceError("boom"))
|
||
# 同报告期 eps 不一致 → 真实性不成立 → 拒绝导入
|
||
fallback = _FakeFinSource(
|
||
name="sina",
|
||
payload=[
|
||
_fin("600519.SH", "2026-03-31", "2026-04-25", eps="1.00", gross="89.759217", source="sina"),
|
||
_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", gross="89.555212", source="sina"),
|
||
],
|
||
)
|
||
syncer, _logs = _fin_syncer(session, primary, fallback)
|
||
res = syncer.sync_symbol("600519.SH")
|
||
assert res.status == "failed"
|
||
assert res.source == "sina"
|
||
assert _count(session, FinancialIndicatorModel) == 1 # 未新增
|
||
|
||
def test_sina_fallback_no_local_history_denied(self, session: Session) -> None:
|
||
primary = _FakeFinSource(name="tushare", error=DataSourceError("频率超限"))
|
||
fallback = _FakeFinSource(
|
||
name="sina",
|
||
payload=[_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", source="sina")],
|
||
)
|
||
syncer, logs = _fin_syncer(session, primary, fallback)
|
||
res = syncer.sync_symbol("600519.SH")
|
||
assert res.status == "failed"
|
||
assert "本地无历史" in " ".join(res.notes)
|
||
assert _count(session, FinancialIndicatorModel) == 0
|
||
assert not logs[-1].success
|
||
|
||
|
||
# ================================================================ 日线同步服务
|
||
|
||
class _FakeDailyTushare:
|
||
name = "tushare"
|
||
|
||
def __init__(self, *, bars=None, factors=None, daily_error=None, factor_error=None) -> None: # noqa: ANN001
|
||
self.bars = bars or []
|
||
self.factors = factors or []
|
||
self.daily_error = daily_error
|
||
self.factor_error = factor_error
|
||
self.daily_calls = 0
|
||
|
||
def get_daily(self, symbol, start, end):
|
||
self.daily_calls += 1
|
||
if self.daily_error is not None:
|
||
raise self.daily_error
|
||
return list(self.bars)
|
||
|
||
def get_adjust_factor(self, symbol, start, end):
|
||
if self.factor_error is not None:
|
||
raise self.factor_error
|
||
return list(self.factors)
|
||
|
||
|
||
class _FakeDailySina:
|
||
name = "sina"
|
||
|
||
def __init__(self, bars) -> None:
|
||
self.bars = bars
|
||
|
||
def get_daily(self, symbol, start, end):
|
||
return [
|
||
b for b in self.bars if start <= b.trade_date <= end
|
||
]
|
||
|
||
|
||
def _daily_syncer(session, primary, fallback=None, *, today: date = TODAY):
|
||
logs: list[SyncLog] = []
|
||
syncer = VerifiedDailySyncer(
|
||
primary=primary,
|
||
fallback=fallback,
|
||
bars=SqlAlchemyDailyBarRepository(session),
|
||
factors=SqlAlchemyAdjustFactorRepository(session),
|
||
audit=logs.append,
|
||
today=today,
|
||
)
|
||
return syncer, logs
|
||
|
||
|
||
class TestVerifiedDailySyncer:
|
||
def test_primary_ok_writes_bars_and_factors(self, session: Session) -> None:
|
||
from app.domain.entities.market import AdjustFactor
|
||
|
||
bars = [_bar("600519.SH", "2026-09-01", "10"), _bar("600519.SH", "2026-09-02", "10")]
|
||
factors = [
|
||
AdjustFactor(symbol="600519.SH", trade_date=date(2026, 9, 1), factor=Decimal("1.0"))
|
||
]
|
||
syncer, logs = _daily_syncer(
|
||
session, _FakeDailyTushare(bars=bars, factors=factors)
|
||
)
|
||
res = syncer.sync_symbol("600519.SH", date(2026, 9, 1), date(2026, 9, 2))
|
||
assert res.status == "ok"
|
||
assert res.bars_written == 2 and res.factors_written == 1
|
||
assert _count(session, StockDailyModel) == 2
|
||
assert _count(session, AdjustFactorModel) == 1
|
||
assert [log.api for log in logs] == ["get_daily", "get_adjust_factor"]
|
||
|
||
def test_factor_failure_aborts_symbol_without_partial_write(self, session: Session) -> None:
|
||
bars = [_bar("600519.SH", "2026-09-01", "10")]
|
||
syncer, _logs = _daily_syncer(
|
||
session,
|
||
_FakeDailyTushare(bars=bars, factor_error=DataSourceError("限速")),
|
||
)
|
||
res = syncer.sync_symbol("600519.SH", date(2026, 9, 1), date(2026, 9, 2))
|
||
assert res.status == "failed"
|
||
# 缺复权因子不写本段日线，避免 resume 漏补因子
|
||
assert _count(session, StockDailyModel) == 0
|
||
|
||
def _seed_local(self, session: Session) -> None:
|
||
repo = SqlAlchemyDailyBarRepository(session)
|
||
repo.upsert_many(
|
||
[
|
||
_bar("600519.SH", "2026-08-31", "10"),
|
||
_bar("600519.SH", "2026-09-01", "10"),
|
||
_bar("600519.SH", "2026-09-02", "10"),
|
||
]
|
||
)
|
||
session.commit()
|
||
|
||
def test_sina_fallback_fills_only_missing_dates(self, session: Session) -> None:
|
||
self._seed_local(session)
|
||
sina_bars = [
|
||
_bar("600519.SH", "2026-08-31", "10", source="sina"),
|
||
_bar("600519.SH", "2026-09-01", "10", source="sina"),
|
||
_bar("600519.SH", "2026-09-02", "10", source="sina"),
|
||
_bar("600519.SH", "2026-09-03", "10", source="sina"),
|
||
_bar("600519.SH", "2026-09-04", "10", source="sina"),
|
||
]
|
||
syncer, logs = _daily_syncer(
|
||
session,
|
||
_FakeDailyTushare(daily_error=DataSourceError("频率超限")),
|
||
fallback=_FakeDailySina(sina_bars),
|
||
)
|
||
res = syncer.sync_symbol("600519.SH", date(2026, 9, 3), date(2026, 9, 4))
|
||
session.commit()
|
||
assert res.status == "sina"
|
||
assert res.bars_written == 2 # 只补 09-03/09-04
|
||
rows = session.scalars(select(StockDailyModel).order_by(StockDailyModel.trade_date)).all()
|
||
assert len(rows) == 5
|
||
gap = [r for r in rows if r.trade_date >= date(2026, 9, 3)]
|
||
assert all(r.source == "sina" and r.adjust == "qfq" for r in gap)
|
||
kept = [r for r in rows if r.trade_date < date(2026, 9, 3)]
|
||
assert all(r.source == "tushare" for r in kept) # 既有行不被覆盖
|
||
assert logs[-1].source == "sina" and logs[-1].success
|
||
|
||
def test_sina_fallback_denied_on_mismatch(self, session: Session) -> None:
|
||
self._seed_local(session)
|
||
# 最近重叠日收盘不一致（新浪 9 元 vs 本地 10 元）→ 通道不可信 → 拒绝
|
||
sina_bars = [_bar("600519.SH", "2026-09-02", "9", source="sina")]
|
||
syncer, _logs = _daily_syncer(
|
||
session,
|
||
_FakeDailyTushare(daily_error=DataSourceError("boom")),
|
||
fallback=_FakeDailySina(sina_bars),
|
||
)
|
||
res = syncer.sync_symbol("600519.SH", date(2026, 9, 3), date(2026, 9, 4))
|
||
assert res.status == "failed"
|
||
assert _count(session, StockDailyModel) == 3 # 未新增
|
||
|
||
def test_sina_fallback_no_overlap_denied(self, session: Session) -> None:
|
||
# 本地没有任何近期历史可对照 → 拒绝新浪补缺
|
||
sina_bars = [_bar("600519.SH", "2026-09-03", "10", source="sina")]
|
||
syncer, logs = _daily_syncer(
|
||
session,
|
||
_FakeDailyTushare(daily_error=DataSourceError("boom")),
|
||
fallback=_FakeDailySina(sina_bars),
|
||
)
|
||
res = syncer.sync_symbol("600519.SH", date(2026, 9, 3), date(2026, 9, 4))
|
||
assert res.status == "failed"
|
||
assert _count(session, StockDailyModel) == 0
|
||
assert not logs[-1].success
|