simon/qlib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(data): 财务/日线同步增量 + 新浪「两边一致」校验兜底 + 逐只进度

Browse Source 
- financial 默认增量：按 A 股披露节奏判断已最新并跳过；--full 强制全量重拉
- Tushare fina_indicator 增加报告期窗口与 100 条/请求自动分页（修复老报告期静默截断）
- 新浪兜底收紧为校验兜底：两源重叠历史一致才导入缺失键，行标记 source=sina；
  财务可比字段取 eps/销售毛利率（ROE 两端口径不同不作依据），日线只比较最近重叠交易日
- CLI 输出逐只进度与导入内容描述（来源/行数/报告期与公告区间），失败股票留待重跑
- financial_indicator 增 source 列（迁移 d3f6c9a21b04）；新增一致性/分页/服务测试
This commit is contained in:
Simon
2026-09-08 21:48:09 +08:00
parent a77d3c13c3
commit 442999f701
17 changed files with 1519 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/infrastructure/data_sources/failover.py
+7 -3
View File
@@ -69,11 +69,15 @@ class FailoverProvider:
fallback_call=lambda: self.fallback.get_adjust_factor(symbol, start, end),
)
def get_financial(self, symbol: str) -> list:
def get_financial(
self, symbol: str, start: date | None = None, end: date | None = None
) -> list:
return self._with_failover(
"get_financial",
primary_call=lambda: self.primary.get_financial(symbol),
fallback_call=lambda: self.fallback.get_financial(symbol),
start=start,
end=end,
primary_call=lambda: self.primary.get_financial(symbol, start, end),
fallback_call=lambda: self.fallback.get_financial(symbol, start, end),
)
# ---- 内部 ----
backend/app/infrastructure/data_sources/sina.py
+18 -3
View File
@@ -128,8 +128,17 @@ class SinaProvider:
# ---- 财务（兜底 Tushare fina_indicator） ----
def get_financial(self, symbol: str) -> list[FinancialIndicator]:
"""新浪关键指标（source=gjzb），含披露日 publish_date → announce_date。"""
def get_financial(
self,
symbol: str,
start: date | None = None,
end: date | None = None,
) -> list[FinancialIndicator]:
"""新浪关键指标（source=gjzb），含披露日 publish_date → announce_date。
新浪不支持按报告期窗口拉取：忽略 start/end 时返回其全部返回的
报告期；传入窗口则按 report_date 客户端过滤（新浪行 source=sina）。
"""
params = {
"paperCode": _to_sina_symbol(symbol),
"source": "gjzb",
@@ -153,6 +162,11 @@ class SinaProvider:
announce = entry.get("publish_date")
if not announce:
continue # 无披露日不可用于研究（防未来函数）
report_day = _to_date(str(rd))
if start is not None and report_day < start:
continue
if end is not None and report_day > end:
continue
fields: dict[str, Decimal | None] = {
"eps": None,
"roe": None,
@@ -167,8 +181,9 @@ class SinaProvider:
rows.append(
FinancialIndicator(
symbol=symbol,
report_date=_to_date(str(rd)),
report_date=report_day,
announce_date=_to_date(str(announce)),
source="sina",
eps=fields["eps"],
roe=fields["roe"],
total_revenue=fields["total_revenue"],
backend/app/infrastructure/data_sources/tushare.py
+37 -4
View File
@@ -10,7 +10,7 @@ from __future__ import annotations
import importlib
import logging
import time
from datetime import date, datetime
from datetime import date, datetime, timedelta
from decimal import Decimal
from typing import Any
@@ -183,9 +183,42 @@ class TushareProvider:
)
return self.normalize_adj_factor(records)
def get_financial(self, symbol: str) -> list[FinancialIndicator]:
records = self._call("fina_indicator", ts_code=symbol)
return self.normalize_financial(records)
def get_financial(
self,
symbol: str,
start: date | None = None,
end: date | None = None,
) -> list[FinancialIndicator]:
"""fina_indicator：报告期窗口 + 100 条/请求上限自动分页。
Tushare 单次请求最多返回 100 条（超出按最新 100 条截断），因此
全量历史必须按报告期窗口回卷分页，否则老报告期会被静默丢弃。
"""
lo = start or date(1990, 1, 1)
hi = end or date.today()
raw: list[dict[str, Any]] = []
while lo <= hi:
batch = self._call(
"fina_indicator",
ts_code=symbol,
start_date=lo.strftime(_TS_DATE),
end_date=hi.strftime(_TS_DATE),
)
raw += batch
if len(batch) < 100:
break
ends = [
datetime.strptime(str(r["end_date"])[:8], _TS_DATE).date()
for r in batch
if r.get("end_date")
]
if not ends:
break
next_hi = min(ends) - timedelta(days=1)
if next_hi < lo: # 无进展保护（边界簇被截断等极端情况）
break
hi = next_hi
return self.normalize_financial(raw)
# ---- 内部 ----
backend/app/infrastructure/persistence/migrations/versions/d3f6c9a21b04_financial_source_marker.py
+33
View File
@@ -0,0 +1,33 @@
"""financial_indicator 增加 source 来源标记
Revision ID: d3f6c9a21b04
Revises: 91c4e27a03fb
Create Date: 2026-09-08
新浪校验兜底导入的财务行带 source=sina（字段可能不全），与 Tushare
首选行区分；现有行回填默认 tushare（SQLite ADD COLUMN 带常量默认值，
不重写现有数据）。AGENT.md §7 数据来源可追溯。
"""
from __future__ import annotations
from collections.abc import Sequence
import sqlalchemy as sa
from alembic import op
revision: str = "d3f6c9a21b04"
down_revision: str | None = "91c4e27a03fb"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.add_column(
"financial_indicator",
sa.Column("source", sa.String(length=16), nullable=False, server_default="tushare"),
)
def downgrade() -> None:
op.drop_column("financial_indicator", "source")
backend/app/infrastructure/persistence/sqlalchemy/models/market.py
+3
View File
@@ -93,6 +93,9 @@ class FinancialIndicatorModel(Base):
symbol: Mapped[str] = mapped_column(String(SYMBOL_LEN), index=True)
report_date: Mapped[date] = mapped_column(Date, index=True)
announce_date: Mapped[date] = mapped_column(Date, index=True)
source: Mapped[str] = mapped_column(
String(16), default="tushare", server_default="tushare"
)
eps: Mapped[Decimal | None] = mapped_column(Numeric(12, 4), nullable=True)
roe: Mapped[Decimal | None] = mapped_column(Numeric(10, 4), nullable=True)
total_revenue: Mapped[Decimal | None] = mapped_column(Numeric(24, 2), nullable=True)
backend/app/infrastructure/persistence/sqlalchemy/repositories/market_impl.py
+19
View File
@@ -241,6 +241,25 @@ class SqlAlchemyFinancialRepository:
def upsert_many(self, rows: Sequence[FinancialIndicator]) -> int:
return _upsert_by_business_key(self._session, FinancialIndicator, rows)
def list_symbol(self, symbol: str) -> list[FinancialIndicator]:
rows = self._session.scalars(
select(FinancialIndicatorModel)
.where(FinancialIndicatorModel.symbol == symbol)
.order_by(FinancialIndicatorModel.announce_date, FinancialIndicatorModel.report_date)
).all()
return [FinancialIndicator.model_validate(r, from_attributes=True) for r in rows]
def has_report_period(self, symbol: str, report_date: date) -> bool:
exists = self._session.scalar(
select(FinancialIndicatorModel.id)
.where(
FinancialIndicatorModel.symbol == symbol,
FinancialIndicatorModel.report_date == report_date,
)
.limit(1)
)
return exists is not None
def list_announced(
self,
symbol: str,