test+fix: 新浪 jsonp 稳健解析(容忍注释/括号包裹/尾部杂字符);Job API 测试改 tmp 种子库

- 真实响应含 /*...*/ 注释与 var data=([...]); 前缀 → _extract_jsonp 取首 '[' 至末 ']'
- 新浪真实网络冒烟通过:财务 100 期(含披露日)+ 日K 最近窗口(source=sina/adjust=qfq)
- TestJobsApi / experiments 用例改为 monkeypatch SessionLocal → tmp 种子库(5 股×300 日),
  与真实全市场库(779 万行)解耦,全量稳定 <1min
- pytest 全量 141 passed / ruff clean
This commit is contained in:
Simon
2026-09-06 21:34:29 +08:00
parent 56254172b3
commit 02e42184be
3 changed files with 80 additions and 13 deletions
@@ -69,10 +69,15 @@ def _to_sina_symbol(symbol: str) -> str:
def _extract_jsonp(payload: str) -> list[dict[str, Any]]: def _extract_jsonp(payload: str) -> list[dict[str, Any]]:
match = re.search(r"=\s*(\[.*\])\s*$", payload.strip(), flags=re.DOTALL) """从 JSONP 中提取数组:容忍前导注释 / var data=([...]) 包裹 / 尾部杂字符。
if not match:
直接取首个 '[' 与末个 ']' 之间的内容(行情数组为扁平结构,无嵌套数组)。
"""
start = payload.find("[")
end = payload.rfind("]")
if start == -1 or end <= start:
raise DataSourceError("新浪行情返回格式无法解析") raise DataSourceError("新浪行情返回格式无法解析")
return json.loads(match.group(1)) return json.loads(payload[start : end + 1])
def _d(value) -> Decimal | None: def _d(value) -> Decimal | None:
+57 -10
View File
@@ -11,7 +11,12 @@ from app.domain.entities.research import (
JobStatus, JobStatus,
ResearchSpec, ResearchSpec,
) )
from app.infrastructure.persistence.sqlalchemy import session as sess_mod
from app.infrastructure.persistence.sqlalchemy.base import Base from app.infrastructure.persistence.sqlalchemy.base import Base
from app.infrastructure.persistence.sqlalchemy.models.market import (
StockDailyModel,
StockModel,
)
from app.infrastructure.persistence.sqlalchemy.repositories.jobs_impl import ( from app.infrastructure.persistence.sqlalchemy.repositories.jobs_impl import (
SqlAlchemyExperimentRepository, SqlAlchemyExperimentRepository,
SqlAlchemyJobRepository, SqlAlchemyJobRepository,
@@ -142,8 +147,42 @@ class TestJobExecutor:
assert done.error assert done.error
@pytest.fixture()
def seeded_api_db(tmp_path, monkeypatch) -> None:
"""把全局 SessionLocal 指向 tmp 种子库(5 只股票 × 300 交易日),
Job 后台执行跑小数据,避免依赖真实全市场库(779 万行)导致过慢。"""
from sqlalchemy.orm import sessionmaker
engine = create_engine(f"sqlite:///{tmp_path / 'api.db'}", future=True)
Base.metadata.create_all(engine)
sf = sessionmaker(bind=engine, expire_on_commit=False)
drifts = {s: 0.003 - 0.0015 * i for i, s in enumerate(_SYMS)}
daily = synthetic_daily(drifts, n=320)
with sf() as session:
for i, sym in enumerate(_SYMS):
session.add(StockModel(symbol=sym, name=f"测试股份{i}", list_date=date(1999, 1, 1)))
rows = [
{
"symbol": r.symbol,
"trade_date": r.trade_date,
"source": "tushare",
"adjust": "none",
"open": float(r.open),
"high": float(r.high),
"low": float(r.low),
"close": float(r.close),
"volume": float(r.volume),
"amount": float(r.amount),
}
for r in daily.itertuples()
]
session.execute(StockDailyModel.__table__.insert(), rows)
session.commit()
monkeypatch.setattr(sess_mod, "SessionLocal", sf)
class TestJobsApi: class TestJobsApi:
def test_submit_then_query(self) -> None: def test_submit_then_query(self, seeded_api_db) -> None:
with TestClient(app) as client: with TestClient(app) as client:
resp = client.post("/api/jobs", json=_spec().model_dump(mode="json")) resp = client.post("/api/jobs", json=_spec().model_dump(mode="json"))
assert resp.status_code == 200 assert resp.status_code == 200
@@ -161,17 +200,25 @@ class TestJobsApi:
assert state["spec"]["factors"][0]["name"] == "momentum_20" assert state["spec"]["factors"][0]["name"] == "momentum_20"
assert state["experiment_id"] assert state["experiment_id"]
def test_unknown_job_404(self) -> None: def test_unknown_job_404(self, seeded_api_db) -> None:
with TestClient(app) as client: with TestClient(app) as client:
assert client.get("/api/jobs/JOB-NOPE").status_code == 404 assert client.get("/api/jobs/JOB-NOPE").status_code == 404
def test_experiments_list_and_detail(self) -> None: def test_experiments_list_and_detail(self, seeded_api_db) -> None:
with TestClient(app) as client: with TestClient(app) as client:
resp = client.get("/api/experiments") # 先跑一个 job 生成归档,再验证列表与详情
resp = client.post("/api/jobs", json=_spec().model_dump(mode="json"))
assert resp.status_code == 200 assert resp.status_code == 200
exps = resp.json() job_id = resp.json()["job_id"]
if exps: # 本机真实库中可能有历史实验;detail 可读即通过 for _ in range(30):
first = exps[0] st = client.get(f"/api/jobs/{job_id}").json()["status"]
detail = client.get(f"/api/experiments/{first['id']}") if st in (JobStatus.SUCCESS, JobStatus.FAILED):
assert detail.status_code == 200 break
assert "result" in detail.json() assert st == JobStatus.SUCCESS
exps = client.get("/api/experiments").json()
assert exps, "job 成功后应有 Experiment 归档"
first = exps[0]
detail = client.get(f"/api/experiments/{first['id']}")
assert detail.status_code == 200
assert "result" in detail.json()
+15
View File
@@ -154,3 +154,18 @@ class TestFormatParity:
# 与 Tushare 同 schema:必备字段齐全 # 与 Tushare 同 schema:必备字段齐全
assert bar.symbol == "600519.SH" assert bar.symbol == "600519.SH"
assert bar.close is not None assert bar.close is not None
class TestJsonpParens:
def test_real_world_paren_wrapped(self) -> None:
payload = 'var data=([{"day":"2024-08-30","close":"1710.0"}])'
rows = _extract_jsonp(payload)
assert rows[0]["close"] == "1710.0"
def test_with_leading_comment_and_trailing_semicolon(self) -> None:
payload = (
"/*<script>location.href='//sina.com';</script>*/\n"
'var data=([{"day":"2024-08-30","close":"1710.0"}]);'
)
rows = _extract_jsonp(payload)
assert rows[0]["close"] == "1710.0"