feat(data): 财务/日线同步增量 + 新浪「两边一致」校验兜底 + 逐只进度

- financial 默认增量：按 A 股披露节奏判断已最新并跳过；--full 强制全量重拉 - Tushare fina_indicator 增加报告期窗口与 100 条/请求自动分页（修复老报告期静默截断） - 新浪兜底收紧为校验兜底：两源重叠历史一致才导入缺失键，行标记 source=sina；财务可比字段取 eps/销售毛利率（ROE 两端口径不同不作依据），日线只比较最近重叠交易日 - CLI 输出逐只进度与导入内容描述（来源/行数/报告期与公告区间），失败股票留待重跑 - financial_indicator 增 source 列（迁移 d3f6c9a21b04）；新增一致性/分页/服务测试
2026-09-08 21:48:09 +08:00
parent a77d3c13c3
commit 442999f701
17 changed files with 1519 additions and 55 deletions
@@ -0,0 +1,450 @@
+"""增量 + 新浪「两边一致」校验兜底同步服务测试。
+
+覆盖：一致性校验纯函数、报告期节奏、财务/日线同步服务
+（temp SQLite + Fake Provider，不触网）。
+"""
+
+from __future__ import annotations
+
+from datetime import date
+from decimal import Decimal
+
+import pytest
+from app.application.services.data_sync import (
+    VerifiedDailySyncer,
+    VerifiedFinancialSyncer,
+    daily_overlap_consistent,
+    financial_overlap_consistent,
+    latest_expected_report_period,
+)
+from app.domain.entities.market import DailyBar, FinancialIndicator, SyncLog
+from app.infrastructure.data_sources.errors import DataSourceError
+from app.infrastructure.persistence.sqlalchemy.base import Base
+from app.infrastructure.persistence.sqlalchemy.models.market import (
+    AdjustFactorModel,
+    FinancialIndicatorModel,
+    StockDailyModel,
+)
+from app.infrastructure.persistence.sqlalchemy.repositories.market_impl import (
+    SqlAlchemyAdjustFactorRepository,
+    SqlAlchemyDailyBarRepository,
+    SqlAlchemyFinancialRepository,
+)
+from sqlalchemy import create_engine, func, select
+from sqlalchemy.orm import Session
+
+TODAY = date(2026, 9, 8)
+DUE = date(2026, 6, 30)  # TODAY 属于半年报季 → 应披露最新报告期
+
+
+@pytest.fixture()
+def session(tmp_path) -> Session:
+    engine = create_engine(f"sqlite:///{tmp_path / 'ds.db'}", future=True)
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        yield session
+
+
+def _fin(
+    symbol: str,
+    report: str,
+    announce: str,
+    *,
+    eps: str = "1.00",
+    roe: str | None = None,
+    gross: str | None = None,
+    source: str = "tushare",
+) -> FinancialIndicator:
+    return FinancialIndicator(
+        symbol=symbol,
+        report_date=date.fromisoformat(report),
+        announce_date=date.fromisoformat(announce),
+        source=source,
+        eps=Decimal(eps),
+        roe=Decimal(roe) if roe is not None else None,
+        gross_margin=Decimal(gross) if gross is not None else None,
+    )
+
+
+def _bar(symbol: str, day: str, close: str, source: str = "tushare") -> DailyBar:
+    return DailyBar(
+        symbol=symbol,
+        trade_date=date.fromisoformat(day),
+        source=source,
+        adjust="none" if source == "tushare" else "qfq",
+        open=Decimal(close),
+        high=Decimal(close),
+        low=Decimal(close),
+        close=Decimal(close),
+        volume=Decimal("1000"),
+    )
+
+
+# ================================================================ 一致性校验
+
+class TestFinancialConsistency:
+    def test_overlap_consistent_passes(self) -> None:
+        local = [
+            _fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", roe="10.5687", gross="89.7592"),
+            _fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", roe="17.9543", gross="89.5552"),
+        ]
+        sina = [
+            _fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", roe="10.57", gross="89.759217", source="sina"),
+            _fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", roe="16.75", gross="89.555212", source="sina"),
+        ]
+        verdict = financial_overlap_consistent(local, sina)
+        # ROE 两边口径不同（新浪加权 vs tushare 摊薄），不参与比较也不判失败
+        assert verdict.ok
+        assert verdict.shared == 2
+        assert not verdict.mismatches
+
+    def test_roe_difference_not_compared(self) -> None:
+        """口径差异字段绝不能导致误杀（17.9543 vs 16.75 是同源数据真实形态）。"""
+        local = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", roe="17.9543", gross="89.5552")]
+        sina = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", roe="16.75", gross="89.555212", source="sina")]
+        verdict = financial_overlap_consistent(local, sina, min_shared=1)
+        assert verdict.ok
+
+    def test_eps_mismatch_fails(self) -> None:
+        local = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", gross="89.5552")]
+        sina = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="99.99", gross="89.555212", source="sina")]
+        verdict = financial_overlap_consistent(local, sina, min_shared=1)
+        assert not verdict.ok
+        assert verdict.mismatches
+
+    def test_insufficient_overlap_fails(self) -> None:
+        local = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57")]
+        sina = [_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", source="sina")]
+        verdict = financial_overlap_consistent(local, sina, min_shared=2)
+        assert not verdict.ok
+        assert verdict.summary()
+
+
+class TestDailyConsistency:
+    def test_recent_overlap_equal_passes(self) -> None:
+        local = [_bar("600519.SH", "2026-09-01", "10"), _bar("600519.SH", "2026-09-02", "10")]
+        sina = [
+            _bar("600519.SH", "2026-09-01", "10", source="sina"),
+            _bar("600519.SH", "2026-09-02", "10", source="sina"),
+        ]
+        verdict = daily_overlap_consistent(local, sina, min_shared=2)
+        assert verdict.ok
+
+    def test_old_dividend_scale_gap_ignored(self) -> None:
+        """新浪前复权：老交易日在除权后数值整体平移，只有「最近重叠段」可比。"""
+        local = [
+            _bar("600519.SH", "2025-06-02", "500"),
+            _bar("600519.SH", "2026-09-01", "10"),
+            _bar("600519.SH", "2026-09-02", "10"),
+            _bar("600519.SH", "2026-09-03", "10"),
+        ]
+        sina = [
+            _bar("600519.SH", "2025-06-02", "480", source="sina"),  # 除权后按因子平移
+            _bar("600519.SH", "2026-09-01", "10", source="sina"),
+            _bar("600519.SH", "2026-09-02", "10", source="sina"),
+            _bar("600519.SH", "2026-09-03", "10", source="sina"),
+        ]
+        verdict = daily_overlap_consistent(local, sina, min_shared=2, max_recent=3)
+        assert verdict.ok
+
+    def test_recent_close_mismatch_fails(self) -> None:
+        local = [_bar("600519.SH", "2026-09-02", "10")]
+        sina = [_bar("600519.SH", "2026-09-02", "9", source="sina")]
+        verdict = daily_overlap_consistent(local, sina, min_shared=1)
+        assert not verdict.ok
+
+
+class TestReportPeriodRhythm:
+    def test_windows(self) -> None:
+        assert latest_expected_report_period(date(2026, 1, 1)) == date(2025, 9, 30)
+        assert latest_expected_report_period(date(2026, 2, 14)) == date(2025, 9, 30)
+        assert latest_expected_report_period(date(2026, 2, 15)) == date(2026, 3, 31)
+        assert latest_expected_report_period(date(2026, 6, 30)) == date(2026, 3, 31)
+        assert latest_expected_report_period(date(2026, 7, 1)) == date(2026, 6, 30)
+        assert latest_expected_report_period(TODAY) == DUE
+        assert latest_expected_report_period(date(2026, 10, 15)) == date(2026, 6, 30)
+        assert latest_expected_report_period(date(2026, 10, 16)) == date(2026, 9, 30)
+
+
+# ================================================================ 财务同步服务
+
+class _FakeFinSource:
+    def __init__(self, *, name: str, payload=None, error: Exception | None = None) -> None:
+        self.name = name
+        self.payload = payload or []
+        self.error = error
+        self.calls: list[tuple] = []
+
+    def get_financial(self, symbol, start=None, end=None):  # noqa: ANN001
+        self.calls.append((symbol, start, end))
+        if self.error is not None:
+            raise self.error
+        return list(self.payload)
+
+
+def _fin_syncer(session, primary, fallback=None, *, today: date = TODAY):
+    logs: list[SyncLog] = []
+    syncer = VerifiedFinancialSyncer(
+        primary=primary,
+        fallback=fallback,
+        repo=SqlAlchemyFinancialRepository(session),
+        audit=logs.append,
+        today=today,
+    )
+    return syncer, logs
+
+
+def _count(session, model) -> int:
+    return session.scalar(select(func.count()).select_from(model))
+
+
+class TestVerifiedFinancialSyncer:
+    def _rows(self, symbol: str = "600519.SH") -> list[FinancialIndicator]:
+        return [
+            _fin(symbol, "2026-03-31", "2026-04-25", eps="21.76", gross="89.7592"),
+            _fin(symbol, "2026-06-30", "2026-08-15", eps="35.57", gross="89.5552"),
+        ]
+
+    def test_first_fetch_writes_delta_and_second_run_skips(self, session: Session) -> None:
+        primary = _FakeFinSource(name="tushare", payload=self._rows())
+        syncer, logs = _fin_syncer(session, primary)
+
+        res = syncer.sync_symbol("600519.SH")
+        session.commit()
+        assert res.status == "ok"
+        assert res.written == 2
+        assert _count(session, FinancialIndicatorModel) == 2
+        assert logs[0].source == "tushare" and logs[0].success
+
+        # 第二次：本地已含最新应披露报告期 2026-06-30 → 跳过，不再调 API
+        res2 = syncer.sync_symbol("600519.SH")
+        assert res2.status == "skip"
+        assert len(primary.calls) == 1
+
+    def test_force_full_overwrites(self, session: Session) -> None:
+        primary = _FakeFinSource(name="tushare", payload=self._rows())
+        syncer, _logs = _fin_syncer(session, primary)
+        syncer.sync_symbol("600519.SH")
+        session.commit()
+
+        changed = [
+            r.model_copy(update={"eps": Decimal("99")}) for r in self._rows()
+        ] + [_fin("600519.SH", "2025-12-31", "2026-04-17", eps="65.66", gross="91.1796")]
+        primary.payload = changed
+        res = syncer.sync_symbol("600519.SH", force_full=True)
+        session.commit()
+        assert res.status == "ok"
+        assert res.written == 3
+        assert res.updated == 2  # 既有 2 行被覆盖
+        got = session.scalars(select(FinancialIndicatorModel)).all()
+        assert len(got) == 3
+        assert max(float(r.eps) for r in got) == 99.0
+
+    def test_sina_fallback_verified_imports_only_new_keys(self, session: Session) -> None:
+        repo = SqlAlchemyFinancialRepository(session)
+        repo.upsert_many(
+            [
+                _fin("600519.SH", "2025-12-31", "2026-04-17", eps="65.66", gross="91.1796"),
+                _fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", gross="89.7592"),
+            ]
+        )
+        session.commit()
+
+        sina_rows = [
+            _fin("600519.SH", "2025-12-31", "2026-04-17", eps="65.66", gross="91.179551", source="sina"),
+            _fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", gross="89.759217", source="sina"),
+            _fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", gross="89.555212", source="sina"),
+        ]
+        primary = _FakeFinSource(name="tushare", error=DataSourceError("频率超限"))
+        fallback = _FakeFinSource(name="sina", payload=sina_rows)
+        syncer, logs = _fin_syncer(session, primary, fallback)
+
+        res = syncer.sync_symbol("600519.SH")
+        session.commit()
+        assert res.status == "sina"
+        assert res.written == 1  # 只补本地缺失键（2026-06-30）
+        rows = session.scalars(select(FinancialIndicatorModel)).all()
+        assert len(rows) == 3
+        new = next(r for r in rows if r.report_date == DUE)
+        assert new.source == "sina"
+        # 审计：tushare 失败 + sina 成功
+        assert [log.source for log in logs] == ["tushare", "sina"]
+        assert logs[0].success is False and logs[1].success is True
+
+    def test_sina_fallback_denied_on_mismatch(self, session: Session) -> None:
+        repo = SqlAlchemyFinancialRepository(session)
+        # 用非「最新应披露期」做种子，避免触发「已最新跳过」
+        repo.upsert_many(
+            [_fin("600519.SH", "2026-03-31", "2026-04-25", eps="21.76", gross="89.7592")]
+        )
+        session.commit()
+
+        primary = _FakeFinSource(name="tushare", error=DataSourceError("boom"))
+        # 同报告期 eps 不一致 → 真实性不成立 → 拒绝导入
+        fallback = _FakeFinSource(
+            name="sina",
+            payload=[
+                _fin("600519.SH", "2026-03-31", "2026-04-25", eps="1.00", gross="89.759217", source="sina"),
+                _fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", gross="89.555212", source="sina"),
+            ],
+        )
+        syncer, _logs = _fin_syncer(session, primary, fallback)
+        res = syncer.sync_symbol("600519.SH")
+        assert res.status == "failed"
+        assert res.source == "sina"
+        assert _count(session, FinancialIndicatorModel) == 1  # 未新增
+
+    def test_sina_fallback_no_local_history_denied(self, session: Session) -> None:
+        primary = _FakeFinSource(name="tushare", error=DataSourceError("频率超限"))
+        fallback = _FakeFinSource(
+            name="sina",
+            payload=[_fin("600519.SH", "2026-06-30", "2026-08-15", eps="35.57", source="sina")],
+        )
+        syncer, logs = _fin_syncer(session, primary, fallback)
+        res = syncer.sync_symbol("600519.SH")
+        assert res.status == "failed"
+        assert "本地无历史" in " ".join(res.notes)
+        assert _count(session, FinancialIndicatorModel) == 0
+        assert not logs[-1].success
+
+
+# ================================================================ 日线同步服务
+
+class _FakeDailyTushare:
+    name = "tushare"
+
+    def __init__(self, *, bars=None, factors=None, daily_error=None, factor_error=None) -> None:  # noqa: ANN001
+        self.bars = bars or []
+        self.factors = factors or []
+        self.daily_error = daily_error
+        self.factor_error = factor_error
+        self.daily_calls = 0
+
+    def get_daily(self, symbol, start, end):
+        self.daily_calls += 1
+        if self.daily_error is not None:
+            raise self.daily_error
+        return list(self.bars)
+
+    def get_adjust_factor(self, symbol, start, end):
+        if self.factor_error is not None:
+            raise self.factor_error
+        return list(self.factors)
+
+
+class _FakeDailySina:
+    name = "sina"
+
+    def __init__(self, bars) -> None:
+        self.bars = bars
+
+    def get_daily(self, symbol, start, end):
+        return [
+            b for b in self.bars if start <= b.trade_date <= end
+        ]
+
+
+def _daily_syncer(session, primary, fallback=None, *, today: date = TODAY):
+    logs: list[SyncLog] = []
+    syncer = VerifiedDailySyncer(
+        primary=primary,
+        fallback=fallback,
+        bars=SqlAlchemyDailyBarRepository(session),
+        factors=SqlAlchemyAdjustFactorRepository(session),
+        audit=logs.append,
+        today=today,
+    )
+    return syncer, logs
+
+
+class TestVerifiedDailySyncer:
+    def test_primary_ok_writes_bars_and_factors(self, session: Session) -> None:
+        from app.domain.entities.market import AdjustFactor
+
+        bars = [_bar("600519.SH", "2026-09-01", "10"), _bar("600519.SH", "2026-09-02", "10")]
+        factors = [
+            AdjustFactor(symbol="600519.SH", trade_date=date(2026, 9, 1), factor=Decimal("1.0"))
+        ]
+        syncer, logs = _daily_syncer(
+            session, _FakeDailyTushare(bars=bars, factors=factors)
+        )
+        res = syncer.sync_symbol("600519.SH", date(2026, 9, 1), date(2026, 9, 2))
+        assert res.status == "ok"
+        assert res.bars_written == 2 and res.factors_written == 1
+        assert _count(session, StockDailyModel) == 2
+        assert _count(session, AdjustFactorModel) == 1
+        assert [log.api for log in logs] == ["get_daily", "get_adjust_factor"]
+
+    def test_factor_failure_aborts_symbol_without_partial_write(self, session: Session) -> None:
+        bars = [_bar("600519.SH", "2026-09-01", "10")]
+        syncer, _logs = _daily_syncer(
+            session,
+            _FakeDailyTushare(bars=bars, factor_error=DataSourceError("限速")),
+        )
+        res = syncer.sync_symbol("600519.SH", date(2026, 9, 1), date(2026, 9, 2))
+        assert res.status == "failed"
+        # 缺复权因子不写本段日线，避免 resume 漏补因子
+        assert _count(session, StockDailyModel) == 0
+
+    def _seed_local(self, session: Session) -> None:
+        repo = SqlAlchemyDailyBarRepository(session)
+        repo.upsert_many(
+            [
+                _bar("600519.SH", "2026-08-31", "10"),
+                _bar("600519.SH", "2026-09-01", "10"),
+                _bar("600519.SH", "2026-09-02", "10"),
+            ]
+        )
+        session.commit()
+
+    def test_sina_fallback_fills_only_missing_dates(self, session: Session) -> None:
+        self._seed_local(session)
+        sina_bars = [
+            _bar("600519.SH", "2026-08-31", "10", source="sina"),
+            _bar("600519.SH", "2026-09-01", "10", source="sina"),
+            _bar("600519.SH", "2026-09-02", "10", source="sina"),
+            _bar("600519.SH", "2026-09-03", "10", source="sina"),
+            _bar("600519.SH", "2026-09-04", "10", source="sina"),
+        ]
+        syncer, logs = _daily_syncer(
+            session,
+            _FakeDailyTushare(daily_error=DataSourceError("频率超限")),
+            fallback=_FakeDailySina(sina_bars),
+        )
+        res = syncer.sync_symbol("600519.SH", date(2026, 9, 3), date(2026, 9, 4))
+        session.commit()
+        assert res.status == "sina"
+        assert res.bars_written == 2  # 只补 09-03/09-04
+        rows = session.scalars(select(StockDailyModel).order_by(StockDailyModel.trade_date)).all()
+        assert len(rows) == 5
+        gap = [r for r in rows if r.trade_date >= date(2026, 9, 3)]
+        assert all(r.source == "sina" and r.adjust == "qfq" for r in gap)
+        kept = [r for r in rows if r.trade_date < date(2026, 9, 3)]
+        assert all(r.source == "tushare" for r in kept)  # 既有行不被覆盖
+        assert logs[-1].source == "sina" and logs[-1].success
+
+    def test_sina_fallback_denied_on_mismatch(self, session: Session) -> None:
+        self._seed_local(session)
+        # 最近重叠日收盘不一致（新浪 9 元 vs 本地 10 元）→ 通道不可信 → 拒绝
+        sina_bars = [_bar("600519.SH", "2026-09-02", "9", source="sina")]
+        syncer, _logs = _daily_syncer(
+            session,
+            _FakeDailyTushare(daily_error=DataSourceError("boom")),
+            fallback=_FakeDailySina(sina_bars),
+        )
+        res = syncer.sync_symbol("600519.SH", date(2026, 9, 3), date(2026, 9, 4))
+        assert res.status == "failed"
+        assert _count(session, StockDailyModel) == 3  # 未新增
+
+    def test_sina_fallback_no_overlap_denied(self, session: Session) -> None:
+        # 本地没有任何近期历史可对照 → 拒绝新浪补缺
+        sina_bars = [_bar("600519.SH", "2026-09-03", "10", source="sina")]
+        syncer, logs = _daily_syncer(
+            session,
+            _FakeDailyTushare(daily_error=DataSourceError("boom")),
+            fallback=_FakeDailySina(sina_bars),
+        )
+        res = syncer.sync_symbol("600519.SH", date(2026, 9, 3), date(2026, 9, 4))
+        assert res.status == "failed"
+        assert _count(session, StockDailyModel) == 0
+        assert not logs[-1].success