diff --git a/CLAUDE-agents.md b/CLAUDE-agents.md index a49be03..df538db 100644 --- a/CLAUDE-agents.md +++ b/CLAUDE-agents.md @@ -33,12 +33,12 @@ results = orch.run_daily() # 5 步流程 ## CLI (`finance/cli/agent_cli.py`) ```bash -python cli/agent_cli.py daily # 5 步流程 -python cli/agent_cli.py picks 15 # 选股 -python cli/agent_cli.py risk # 风险评估 -python cli/agent_cli.py research # 因子研究 -python cli/agent_cli.py report 20260603 # 生成日报 -python cli/agent_cli.py warmup 50 # 首次预热缓存 +python finance/cli/agent_cli.py daily # 5 步流程 +python finance/cli/agent_cli.py picks 15 # 选股 +python finance/cli/agent_cli.py risk # 风险评估 +python finance/cli/agent_cli.py research # 因子研究 +python finance/cli/agent_cli.py report 20260603 # 生成日报 +python finance/cli/agent_cli.py warmup 50 # 首次预热缓存 ``` ## Demo 脚本 (`finance/cli/demo_*.py`) diff --git a/CLAUDE.md b/CLAUDE.md index 8c2769c..0cc763f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,13 +10,26 @@ cc-cursor — Mac Mini 单机量化研究平台。全链路:Data → Factor | 因子引擎 + 情绪 | `CLAUDE-factors.md` | `from factors.registry import get_factor` | | 回测 + 优化 | `CLAUDE-backtest.md` | `from backtest.vectorbt.engine import VectorBTEngine` | | ML 模型 | `CLAUDE-ml.md` | `from models.lightgbm.model import LightGBMModel` | -| Agent 系统 + CLI | `CLAUDE-agents.md` | `python cli/agent_cli.py daily` | +| Agent 系统 + CLI | `CLAUDE-agents.md` | `python finance/cli/agent_cli.py daily` | + +## 工作区布局 + +| 目录 | 内容 | +|------|------| +| `finance/` | 核心量化引擎(**代码实际位置**)。代码内 import 用顶层名 `data.*`/`factors.*` 等 — 由 CLI 把 `finance/` 加入 sys.path;文件路径为 `finance/data/xxx.py` 等 | +| `djapi/` | Django API 子项目,有独立 `djapi/CLAUDE.md` | +| `shared/script/` | `autossh.sh` — MariaDB SSH 隧道 | +| `docs/` | `usage.md` / `usage.html` 使用指南、`news_report_api.md` 新闻接口文档;**新建 md 一律放这里** | +| `finance/strategy` `portfolio` `execution` `scheduler/` | 空壳占位(仅 `__init__.py`),逻辑未落地,别误以为有实现 | +| `finance/reports/` | 日报输出 `daily_YYYYMMDD.{md,html}` | ## 环境 ```bash conda activate quant # Python 3.11.13 -bash shared/script/autossh.sh # DB SSH 隧道 +bash shared/script/autossh.sh # DB SSH 隧道 (本地 13306 → 远程 3306) + +# 环境变量在 finance/.env(示例见 finance/.env.example):QWEN / TUSHARE / DB / 情绪范围 ``` ## 任务→文档路由 @@ -28,7 +41,7 @@ bash shared/script/autossh.sh # DB SSH 隧道 | 回测/优化/策略相关 | `CLAUDE-backtest.md` | | ML 模型/特征工程相关 | `CLAUDE-ml.md` | | Agent/CLI/报告相关 | `CLAUDE-agents.md` | -| API 参数/第三方库 | 调用 `mcp__mcphub__context7-query-docs` | +| 第三方库 API/参数 | `web_fetch` / `research` 查官方文档 | | 因子名/类名/表结构速查 | `CLAUDE-reference.md` | ## 多步任务规则 @@ -51,18 +64,20 @@ bash shared/script/autossh.sh # DB SSH 隧道 ## CLI 常用命令 ```bash -python cli/agent_cli.py daily # 5步完整流程 -python cli/agent_cli.py picks 15 # 选股 -python cli/agent_cli.py risk # 风险评估 -python cli/agent_cli.py report 20260603 # 生成日报 -python cli/agent_cli.py warmup 50 # 首次预热缓存 +python finance/cli/agent_cli.py daily # 5步完整流程 +python finance/cli/agent_cli.py picks 15 # 选股 +python finance/cli/agent_cli.py risk # 风险评估 +python finance/cli/agent_cli.py research # 因子研究 +python finance/cli/agent_cli.py report 20260603 # 生成日报 +python finance/cli/agent_cli.py warmup 50 # 首次预热缓存 -python cli/demo_data_manager.py --ts_code 600519.SH -python cli/demo_factor_engine.py --ts_code 300750.SZ -python cli/demo_backtest.py --ts_code 000001.SZ -python cli/demo_optimizer.py --ts_code 000001.SZ --trials 100 -python cli/demo_ml.py --ts_code 000001.SZ --lookahead 5 -python cli/demo_sentiment_detail.py --ts_code 600519.SH --date 20260603 +python finance/cli/demo_data_manager.py --ts_code 600519.SH +python finance/cli/demo_factor_engine.py --ts_code 300750.SZ +python finance/cli/demo_backtest.py --ts_code 000001.SZ +python finance/cli/demo_optimizer.py --ts_code 000001.SZ --trials 100 +python finance/cli/demo_ml.py --ts_code 000001.SZ --lookahead 5 +python finance/cli/demo_sentiment.py --ts_code 600519.SH +python finance/cli/demo_sentiment_detail.py --ts_code 600519.SH --date 20260603 ``` ## 技术栈 diff --git a/djapi/.env.example b/djapi/.env.example index 231355e..6c373a7 100644 --- a/djapi/.env.example +++ b/djapi/.env.example @@ -12,6 +12,14 @@ MYSQL_USER=myquant MYSQL_PASSWORD=your-mysql-password MYSQL_DATABASE=myquant +# 日报结构化入库 (news_report / news_event) 只读查询 +# 与 report_db_design.md §7 一致;密码必填,缺失时接口直接报错 +NEWS_DB_HOST=127.0.0.1 +NEWS_DB_PORT=3306 +NEWS_DB_USER=myquant +NEWS_DB_PASSWORD=your-news-db-password +NEWS_DB_NAME=myquant + # DeepSeek AI DEEPSEEK_API_KEY=your-deepseek-api-key diff --git a/djapi/README.md b/djapi/README.md index 0f156c1..f224c6d 100644 --- a/djapi/README.md +++ b/djapi/README.md @@ -95,6 +95,10 @@ uwsgi --stop uwsgi.pid | `stockmargin/` | tscode, start_date, end_date | 个股融资融券 | | `xwlbNews/` | start_date, end_date | 新闻联播(原始识别文本) | | `xwlbFine/` | start_date, end_date | 新闻联播(AI 分割后) | +| `news/reports/` | report_type, start_date, end_date, id | 日报查询(默认最近 24h;传 id 返回单份详情含事件) | +| `news/events/` | days, importance, report_type, section, limit | 重要事件聚合(跨日报,最近 N 天 importance≥阈值) | + +日报查询接口详细说明见 [`docs/news_report_api.md`](../docs/news_report_api.md)(表结构见 `djapi/docs/db_schema.md`)。 API 文档(Swagger):`/api/docs/` OpenAPI Schema:`/api/schema/` diff --git a/djapi/api/report/__init__.py b/djapi/api/report/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/djapi/api/report/query.py b/djapi/api/report/query.py new file mode 100644 index 0000000..a15f567 --- /dev/null +++ b/djapi/api/report/query.py @@ -0,0 +1,168 @@ +""" +news_report / news_event 只读查询层(日报结构化入库,见 docs/db_schema.md)。 + +连接配置来自环境变量(与 docs/report_db_design.md §7 保持一致): + NEWS_DB_HOST / NEWS_DB_PORT / NEWS_DB_USER / NEWS_DB_PASSWORD / NEWS_DB_NAME +NEWS_DB_PASSWORD 缺失时直接报错,禁止默认密码。 + +所有 SQL 均为 MariaDB 方言、参数化查询(防 SQL 注入),不依赖 ORM。 +""" + +import json +import os +from datetime import date, datetime, timedelta + +import mysql.connector + +_REPORT_FIELDS = ( + "id, report_date, report_type, file_name, generated_at, " + "ai_summary, stats, created_at" +) +_EVENT_FIELDS = ( + "id, report_id, section, rank, importance, event_type, title, " + "summary, sentiment, source, url" +) + + +def load_db_config() -> dict: + """从环境变量读取 NEWS_DB_* 连接配置,密码缺失时抛错。""" + password = os.getenv("NEWS_DB_PASSWORD") + if not password: + raise RuntimeError( + "NEWS_DB_PASSWORD 未设置,禁止使用默认密码连接 news 库" + ) + return { + "host": os.getenv("NEWS_DB_HOST", "127.0.0.1"), + "port": int(os.getenv("NEWS_DB_PORT", "3306")), + "user": os.getenv("NEWS_DB_USER", "myquant"), + "password": password, + "database": os.getenv("NEWS_DB_NAME", "myquant"), + "charset": "utf8mb4", + } + + +def _connect(): + return mysql.connector.connect(**load_db_config()) + + +def _row_to_dict(row: dict) -> dict: + """序列化行:stats JSON 解析、日期/时间转 ISO 字符串。""" + d = dict(row) + if d.get("stats") is not None: + try: + d["stats"] = json.loads(d["stats"]) + except (TypeError, ValueError): + d["stats"] = None + for k, v in d.items(): + if isinstance(v, (date, datetime)): + d[k] = v.isoformat() + return d + + +def fetch_reports( + report_type: str | None = None, + start_date: date | None = None, + end_date: date | None = None, + report_id: int | None = None, +): + """ + 日报查询。 + + report_id 给定 → 返回单份详情 dict(含 events,按 section, rank 排序); + 不存在返回 None。 + 否则 → 返回列表:每天每类型取最新一份(MAX(generated_at) 子查询), + 仅主表字段(轻量,不带 events)。 + """ + conn = _connect() + try: + cur = conn.cursor(dictionary=True) + if report_id is not None: + cur.execute( + f"SELECT {_REPORT_FIELDS} FROM news_report WHERE id = %s", + (report_id,), + ) + row = cur.fetchone() + if row is None: + return None + report = _row_to_dict(row) + cur.execute( + "SELECT id, section, rank, importance, event_type, title, " + "summary, sentiment, source, url " + "FROM news_event WHERE report_id = %s ORDER BY section, rank", + (report_id,), + ) + report["events"] = [dict(r) for r in cur.fetchall()] + return report + + where, params = [], [] + if report_type: + where.append("r.report_type = %s") + params.append(report_type) + if start_date: + where.append("r.report_date >= %s") + params.append(start_date.isoformat()) + if end_date: + where.append("r.report_date <= %s") + params.append(end_date.isoformat()) + cond = (" WHERE " + " AND ".join(where)) if where else "" + + sql = ( + "SELECT r.id, r.report_date, r.report_type, r.file_name, " + "r.generated_at, r.ai_summary, r.stats, r.created_at " + "FROM news_report r " + "JOIN (" + " SELECT report_date, report_type, MAX(generated_at) AS g " + " FROM news_report GROUP BY report_date, report_type" + ") t ON r.report_date = t.report_date " + " AND r.report_type = t.report_type " + " AND r.generated_at = t.g" + + cond + + " ORDER BY r.report_date DESC, r.report_type" + ) + cur.execute(sql, tuple(params)) + return [_row_to_dict(r) for r in cur.fetchall()] + finally: + conn.close() + + +def fetch_important_events( + days: int = 7, + importance: int = 4, + report_type: str | None = None, + section: str | None = None, + limit: int = 100, +) -> list: + """ + 跨日报重要事件聚合检索(最近 N 天,importance >= 阈值)。 + + 按 importance DESC, report_date DESC 排序。 + """ + conn = _connect() + try: + cur = conn.cursor(dictionary=True) + since = (date.today() - timedelta(days=days)).isoformat() + where = ["r.report_date >= %s", "e.importance >= %s"] + params = [since, int(importance)] + if report_type: + where.append("r.report_type = %s") + params.append(report_type) + if section: + where.append("e.section = %s") + params.append(section) + + sql = ( + "SELECT r.report_date, r.report_type, e.id, e.section, e.rank, " + "e.importance, e.event_type, e.title, e.summary, e.sentiment, " + "e.source, e.url " + "FROM news_event e " + "JOIN news_report r ON r.id = e.report_id " + "WHERE " + " AND ".join(where) + + " ORDER BY e.importance DESC, r.report_date DESC, " + "e.section, e.rank " + + "LIMIT %s" + ) + params.append(int(limit)) + cur.execute(sql, tuple(params)) + return [dict(r) for r in cur.fetchall()] + finally: + conn.close() diff --git a/djapi/api/report/serializers.py b/djapi/api/report/serializers.py new file mode 100644 index 0000000..6c9f4db --- /dev/null +++ b/djapi/api/report/serializers.py @@ -0,0 +1,50 @@ +"""日报查询 API 的 OpenAPI 文档 serializer(只读,不用于反序列化)。""" + +from rest_framework import serializers + + +class EventSerializer(serializers.Serializer): + """日报事件明细(news_event 一行)""" + id = serializers.IntegerField() + section = serializers.CharField() + rank = serializers.IntegerField() + importance = serializers.IntegerField(allow_null=True) + event_type = serializers.CharField(allow_null=True) + title = serializers.CharField() + summary = serializers.CharField(allow_null=True) + sentiment = serializers.CharField(allow_null=True) + source = serializers.CharField(allow_null=True) + url = serializers.CharField(allow_null=True) + + +class ReportListSerializer(serializers.Serializer): + """日报列表项(news_report 主表字段)""" + id = serializers.IntegerField() + report_date = serializers.CharField() + report_type = serializers.CharField() + file_name = serializers.CharField() + generated_at = serializers.CharField() + ai_summary = serializers.CharField(allow_null=True) + stats = serializers.JSONField(allow_null=True) + created_at = serializers.CharField() + + +class ReportDetailSerializer(ReportListSerializer): + """日报详情(主表字段 + 事件列表)""" + events = EventSerializer(many=True) + + +class ImportantEventSerializer(serializers.Serializer): + """跨日报重要事件聚合(news_event JOIN news_report)""" + report_date = serializers.CharField() + report_type = serializers.CharField() + id = serializers.IntegerField() + section = serializers.CharField() + rank = serializers.IntegerField() + importance = serializers.IntegerField(allow_null=True) + event_type = serializers.CharField(allow_null=True) + title = serializers.CharField() + summary = serializers.CharField(allow_null=True) + sentiment = serializers.CharField(allow_null=True) + source = serializers.CharField(allow_null=True) + url = serializers.CharField(allow_null=True) diff --git a/djapi/api/report/tests.py b/djapi/api/report/tests.py new file mode 100644 index 0000000..74b8a9c --- /dev/null +++ b/djapi/api/report/tests.py @@ -0,0 +1,149 @@ +""" +日报查询 API 测试(GET /api/news/reports/ 与 /api/news/events/)。 + +通过 mock 数据库查询层(api.report.query.*),只验证视图层 +参数解析 / 默认窗口 / 响应结构 / 错误处理,不依赖真实 MySQL。 +""" + +from datetime import timedelta +from unittest.mock import patch + +from django.test import TestCase +from django.utils import timezone +from rest_framework.test import APIClient + + +class NewsReportsAPITest(TestCase): + """GET /api/news/reports/ 日报查询""" + + def setUp(self): + self.client = APIClient() + self.url = '/api/news/reports/' + + @patch('api.report.query.fetch_reports', return_value=[]) + def test_default_window_last_24h(self, mock_fetch): + resp = self.client.get(self.url) + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.json(), []) + kwargs = mock_fetch.call_args.kwargs + now = timezone.now() + self.assertEqual(kwargs['start_date'], (now - timedelta(hours=24)).date()) + self.assertEqual(kwargs['end_date'], now.date()) + self.assertIsNone(kwargs['report_type']) + self.assertIsNone(kwargs['report_id']) + + @patch('api.report.query.fetch_reports', + return_value=[{'id': 1, 'report_date': '2026-08-01', 'report_type': 'finance'}]) + def test_report_type_and_date_range(self, mock_fetch): + resp = self.client.get(self.url, { + 'report_type': 'finance', + 'start_date': '2026-07-01', + 'end_date': '2026-08-03', + }) + self.assertEqual(resp.status_code, 200) + self.assertEqual(len(resp.json()), 1) + kwargs = mock_fetch.call_args.kwargs + self.assertEqual(kwargs['report_type'], 'finance') + self.assertEqual(str(kwargs['start_date']), '2026-07-01') + self.assertEqual(str(kwargs['end_date']), '2026-08-03') + + @patch('api.report.query.fetch_reports') + def test_detail_by_id(self, mock_fetch): + mock_fetch.return_value = { + 'id': 10, 'report_date': '2026-08-03', 'report_type': 'finance', + 'events': [{'id': 1, 'title': '事件一'}], + } + resp = self.client.get(self.url, {'id': '10'}) + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.json()['id'], 10) + self.assertEqual(mock_fetch.call_args.kwargs['report_id'], 10) + + @patch('api.report.query.fetch_reports', return_value=None) + def test_detail_not_found(self, mock_fetch): + resp = self.client.get(self.url, {'id': '99999'}) + self.assertEqual(resp.status_code, 404) + self.assertIn('error', resp.json()) + + def test_invalid_report_type(self): + resp = self.client.get(self.url, {'report_type': 'xxx'}) + self.assertEqual(resp.status_code, 400) + + def test_invalid_date_format(self): + resp = self.client.get(self.url, {'start_date': '2026/07/01'}) + self.assertEqual(resp.status_code, 400) + + def test_invalid_id(self): + resp = self.client.get(self.url, {'id': 'abc'}) + self.assertEqual(resp.status_code, 400) + + def test_non_positive_id(self): + resp = self.client.get(self.url, {'id': '0'}) + self.assertEqual(resp.status_code, 400) + + @patch('api.report.query.fetch_reports', side_effect=RuntimeError('db down')) + def test_query_error_500(self, mock_fetch): + resp = self.client.get(self.url) + self.assertEqual(resp.status_code, 500) + self.assertIn('error', resp.json()) + + +class NewsEventsAPITest(TestCase): + """GET /api/news/events/ 重要事件聚合""" + + def setUp(self): + self.client = APIClient() + self.url = '/api/news/events/' + + @patch('api.report.query.fetch_important_events', return_value=[]) + def test_defaults(self, mock_fetch): + resp = self.client.get(self.url) + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.json(), []) + kwargs = mock_fetch.call_args.kwargs + self.assertEqual(kwargs['days'], 7) + self.assertEqual(kwargs['importance'], 4) + self.assertEqual(kwargs['limit'], 100) + self.assertIsNone(kwargs['report_type']) + self.assertIsNone(kwargs['section']) + + @patch('api.report.query.fetch_important_events', + return_value=[{'id': 1, 'title': '重要事件', 'importance': 5}]) + def test_filters(self, mock_fetch): + resp = self.client.get(self.url, { + 'days': '3', 'importance': '5', + 'report_type': 'intl', 'section': 'intl', 'limit': '10', + }) + self.assertEqual(resp.status_code, 200) + self.assertEqual(len(resp.json()), 1) + kwargs = mock_fetch.call_args.kwargs + self.assertEqual(kwargs['days'], 3) + self.assertEqual(kwargs['importance'], 5) + self.assertEqual(kwargs['limit'], 10) + self.assertEqual(kwargs['report_type'], 'intl') + self.assertEqual(kwargs['section'], 'intl') + + def test_invalid_days(self): + resp = self.client.get(self.url, {'days': 'abc'}) + self.assertEqual(resp.status_code, 400) + + def test_days_out_of_range(self): + resp = self.client.get(self.url, {'days': '0'}) + self.assertEqual(resp.status_code, 400) + + def test_invalid_importance(self): + resp = self.client.get(self.url, {'importance': '9'}) + self.assertEqual(resp.status_code, 400) + + def test_invalid_section(self): + resp = self.client.get(self.url, {'section': 'foo'}) + self.assertEqual(resp.status_code, 400) + + def test_invalid_report_type(self): + resp = self.client.get(self.url, {'report_type': 'xxx'}) + self.assertEqual(resp.status_code, 400) + + @patch('api.report.query.fetch_important_events', side_effect=RuntimeError('db down')) + def test_query_error_500(self, mock_fetch): + resp = self.client.get(self.url) + self.assertEqual(resp.status_code, 500) + self.assertIn('error', resp.json()) diff --git a/djapi/api/report/views.py b/djapi/api/report/views.py new file mode 100644 index 0000000..e0929fc --- /dev/null +++ b/djapi/api/report/views.py @@ -0,0 +1,147 @@ +""" +日报查询 API 视图。 + +- GET /api/news/reports/ 日报查询(默认最近 24 小时;传 id 返回单份详情含事件) +- GET /api/news/events/ 跨日报重要事件聚合(最近 N 天 importance >= 阈值) +""" + +from datetime import datetime, timedelta + +from django.utils import timezone +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework.decorators import api_view +from rest_framework.response import Response + +from . import query as report_query +from .serializers import ( + ImportantEventSerializer, + ReportDetailSerializer, + ReportListSerializer, +) + +_REPORT_TYPES = ("finance", "intl") +_SECTIONS = ("xwlb", "news", "cninfo", "intl") + + +def _parse_date_param(request, name): + """解析 YYYY-MM-DD 参数,非法格式抛 ValueError。""" + raw = request.GET.get(name) + if not raw: + return None + try: + return datetime.strptime(raw, "%Y-%m-%d").date() + except ValueError: + raise ValueError(f"{name} 格式错误,应为 YYYY-MM-DD") + + +@extend_schema( + parameters=[ + OpenApiParameter(name='report_type', type=str, required=False, + description='日报类型:finance | intl(默认两者)'), + OpenApiParameter(name='start_date', type=str, required=False, + description='起始日期 YYYY-MM-DD(默认:当前时间往前 24 小时)'), + OpenApiParameter(name='end_date', type=str, required=False, + description='结束日期 YYYY-MM-DD(默认今天)'), + OpenApiParameter(name='id', type=int, required=False, + description='日报 id,指定时返回单份详情(含事件,按板块/序号排序)'), + ], + responses={200: ReportDetailSerializer}, + description='AI 财经日报查询:默认返回最近 24 小时的日报列表(每天每类型取最新一份);' + '传 id 返回单份详情含事件', + tags=['日报'], +) +@api_view(['GET']) +def news_reports(request): + try: + report_id = request.GET.get('id') + if report_id is not None: + report_id = int(report_id) + if report_id <= 0: + raise ValueError("id 必须为正整数") + report_type = request.GET.get('report_type') + if report_type and report_type not in _REPORT_TYPES: + raise ValueError("report_type 仅支持 finance / intl") + start_date = _parse_date_param(request, 'start_date') + end_date = _parse_date_param(request, 'end_date') + except ValueError as e: + return Response({'error': str(e)}, status=400) + + now = timezone.now() + start_date = start_date or (now - timedelta(hours=24)).date() + end_date = end_date or now.date() + + try: + data = report_query.fetch_reports( + report_type=report_type, + start_date=start_date, + end_date=end_date, + report_id=report_id, + ) + except Exception as e: + return Response({'error': f'查询失败: {e}'}, status=500) + + if report_id is not None: + if data is None: + return Response({'error': f'日报 id={report_id} 不存在'}, status=404) + return Response(data) + return Response(data) + + +def _get_int_param(request, name, default, lo, hi): + """解析整数参数并校验范围,非法抛 ValueError。""" + raw = request.GET.get(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError: + raise ValueError(f"{name} 必须为整数") + if not (lo <= value <= hi): + raise ValueError(f"{name} 需在 {lo}~{hi} 之间") + return value + + +@extend_schema( + parameters=[ + OpenApiParameter(name='days', type=int, required=False, default=7, + description='最近 N 天(1~365)'), + OpenApiParameter(name='importance', type=int, required=False, default=4, + description='最低重要度(1~5)'), + OpenApiParameter(name='report_type', type=str, required=False, + description='日报类型:finance | intl(默认两者)'), + OpenApiParameter(name='section', type=str, required=False, + description='板块:xwlb | news | cninfo | intl(默认全部)'), + OpenApiParameter(name='limit', type=int, required=False, default=100, + description='返回条数上限(1~500)'), + ], + responses={200: ImportantEventSerializer(many=True)}, + description='跨日报重要事件聚合:最近 N 天 importance >= 阈值的事件,' + '按重要度、日期降序', + tags=['日报'], +) +@api_view(['GET']) +def news_events(request): + try: + days = _get_int_param(request, 'days', 7, 1, 365) + importance = _get_int_param(request, 'importance', 4, 1, 5) + limit = _get_int_param(request, 'limit', 100, 1, 500) + report_type = request.GET.get('report_type') + if report_type and report_type not in _REPORT_TYPES: + raise ValueError("report_type 仅支持 finance / intl") + section = request.GET.get('section') + if section and section not in _SECTIONS: + raise ValueError("section 仅支持 xwlb / news / cninfo / intl") + except ValueError as e: + return Response({'error': str(e)}, status=400) + + try: + data = report_query.fetch_important_events( + days=days, + importance=importance, + report_type=report_type, + section=section, + limit=limit, + ) + except Exception as e: + return Response({'error': f'查询失败: {e}'}, status=500) + return Response(data) diff --git a/djapi/api/urls.py b/djapi/api/urls.py index c4b24bd..a65920c 100644 --- a/djapi/api/urls.py +++ b/djapi/api/urls.py @@ -1,5 +1,6 @@ from django.urls import path from . import views +from .report import views as report_views urlpatterns = [ # 其他 URL 路由 @@ -21,4 +22,8 @@ urlpatterns = [ path('xwlbNews/', views.xwlbNews, name='xwlbNews'), path('xwlbFine/', views.xwlbFine, name='xwlbFine'), + + # 日报查询(news_report / news_event) + path('news/reports/', report_views.news_reports, name='news_reports'), + path('news/events/', report_views.news_events, name='news_events'), ] \ No newline at end of file diff --git a/djapi/continuation.md b/djapi/continuation.md index 9460cb6..08030f4 100644 --- a/djapi/continuation.md +++ b/djapi/continuation.md @@ -4,6 +4,12 @@ djapi — Django 5.2 金融数据 API 项目,2026-06-17 已部署。 +## Checkpoint 记录 + +| 日期 | 内容 | +|------|------| +| 2026-08-03 | 新增日报查询 API ×2(news/reports/ + news/events/),基于 news_report/news_event 表,已部署 doorcome ✅ | + 服务器:`simon@doorcome.cn`,路径 `/home/simon/myquant/djapi/`,虚拟环境 `/opt/miniconda/envs/django/`。 ## 已完成 @@ -56,7 +62,7 @@ djapi — Django 5.2 金融数据 API 项目,2026-06-17 已部署。 | `main_videos.py` | 批量补缺(扫描缺失日期) | | `mysqlHandle.py` | MySQLDB 重新导出 | -## 所有 API 端点(14 个) +## 所有 API 端点(16 个) | 端点 | 数据源 | 说明 | |------|--------|------| @@ -74,6 +80,32 @@ djapi — Django 5.2 金融数据 API 项目,2026-06-17 已部署。 | `getdiv/` | Tushare | 股息率(TTM rolling + 毛刺平滑) | | `xwlbNews/` | MySQL | 新闻联播原始文本 | | `xwlbFine/` | MySQL | 新闻联播 AI 精编 | +| `news/reports/` | MySQL (news_) | 日报查询:默认最近 24h;传 id 返回详情含事件 | +| `news/events/` | MySQL (news_) | 重要事件聚合:最近 N 天 importance≥阈值 | + +--- + +## 日报查询 API(2026-08-03 新增) + +### 模块 + +- 新增 `api/report/` 包(独立于 stock):`query.py`(连库+查询 SQL)/ `views.py`(2 视图)/ `serializers.py`(OpenAPI)/ `tests.py`(17 个单测,mock 查询层) +- `api/urls.py` 注册 `news/reports/`、`news/events/`;`settings.py` SPECTACULAR TAGS 加「日报」 +- 数据库:doorcome 本机 MariaDB `myquant` 库 `news_report`(180 行)+ `news_event`(4372 条),与现有 `MYSQL_*` 同库同用户 +- 连接配置:服务器 `djapi/.env` 新增 `NEWS_DB_*`(复用 MYSQL_* 值,密码必填否则 500) +- 文档:`docs/news_report_api.md`(使用手册,含线上地址/curl/真实样例);README API 概览表已加两行 + +### 部署(2026-08-03 完成) + +- rsync 增量同步(**未用文档中的 --delete**,见下)→ 重启 uWSGI → 冒烟通过(列表/详情/聚合/400/404) +- 线上:`https://api.doorcome.cn/api/news/reports/`、`/api/news/events/`,Swagger `/api/docs/`「日报」tag + +### 已知事项 + +1. **服务器顶层历史平铺文件未清理**:`/home/simon/myquant/djapi/` 顶层有 views.py/urls.py/smoothBrush.py/getStockDiv2.py/env_loader.py/akshare_data.py(历史 rsync 陷阱产物),`--delete` 会删除它们,但 `divSearch.py`(离线脚本)仍绝对导入顶层 getStockDiv2/smoothBrush → 本次增量同步保留;清理前需先修 divSearch.py 的导入 +2. **既有失败测试**:`api.tests.DateFormatCorrectionTest.test_empty_string`(date_format_correction('') 期望 None 实得 ''),与本次无关 +3. 冒烟曾发现 fetch_reports 列表 SQL 缺 `r.` 别名前缀(1052 ambiguous),已修复 +4. 本地验证需绕过 macOS TCC:`HOME=/tmp/djtest_home PYTHONPATH=/tmp/djtest_pkgs`(tushare 写 ~/tk.csv 被拦 + quant 环境缺 mysql-connector-python) ## 部署 diff --git a/djapi/djapi/settings.py b/djapi/djapi/settings.py index d35a605..3d22d96 100644 --- a/djapi/djapi/settings.py +++ b/djapi/djapi/settings.py @@ -113,6 +113,7 @@ SPECTACULAR_SETTINGS = { {'name': '指数', 'description': '指数行情与查询'}, {'name': '融资融券', 'description': '融资融券明细与汇总'}, {'name': '新闻联播', 'description': '新闻联播 ASR 转写与 AI 精编'}, + {'name': '日报', 'description': 'AI 财经日报查询(news_report / news_event)'}, {'name': '系统', 'description': '系统信息'}, ], } diff --git a/djapi/docs/db_schema.md b/djapi/docs/db_schema.md new file mode 100644 index 0000000..45bc28b --- /dev/null +++ b/djapi/docs/db_schema.md @@ -0,0 +1,111 @@ +# 日报结构化入库:数据库表结构与数据契约 + +> 版本:v1.0 | 2026-08-03 +> 用途:供 API / 前端对接读取日报数据。表位于 MySQL `myquant` 库,表前缀 `news_`。 +> 连接:`192.168.1.10:13306`(pi 上 autossh 隧道 → doorcome.cn:3306 MariaDB 10.11),用户 `myquant`(密码在服务器 `.env` 的 `NEWS_DB_PASSWORD`)。 + +--- + +## 1. 表结构 + +### 1.1 news_report(日报主表,一行 = 一份日报) + +| 字段 | 类型 | 说明 | +| --- | --- | --- | +| id | BIGINT UNSIGNED PK | 自增主键 | +| report_date | DATE | 日报日期 | +| report_type | VARCHAR(16) | `finance`=A 股日报 / `intl`=国际财经日报 | +| file_name | VARCHAR(160) | 历史文件源文件名;**新生成日报为空字符串 `""`** | +| generated_at | DATETIME | 生成时间 | +| ai_summary | TEXT | AI 摘要全文(含换行,按条目分行) | +| stats | JSON | 数据总览统计快照(见第 3 节),可为 NULL | +| created_at | DATETIME | 入库时间 | + +唯一键:`(report_date, report_type, file_name)` —— 历史同一天多次生成(intl 一日 3 次)保留多行;新生成日报 `file_name=''` 每天每类型仅一行,重复生成覆盖。 + +### 1.2 news_event(日报事件明细,一行 = 一条事件) + +| 字段 | 类型 | 说明 | +| --- | --- | --- | +| id | BIGINT UNSIGNED PK | 自增主键 | +| report_id | BIGINT UNSIGNED | FK → news_report.id | +| section | VARCHAR(16) | 板块:`xwlb`=新闻联播 / `news`=财经新闻 / `cninfo`=公告调研 / `intl`=国际重要事件 | +| rank | INT | 板块内序号(1 起) | +| importance | INT NULL | 重要度 1-5 | +| event_type | VARCHAR(64) NULL | 事件类型(如 宏观经济/地缘政治/新闻联播/公告) | +| title | VARCHAR(512) | 标题 | +| summary | TEXT NULL | 摘要/正文 | +| sentiment | VARCHAR(8) NULL | `positive` / `negative` / `neutral` | +| source | VARCHAR(64) NULL | 来源(如 `cls`、`investinglive.com`) | +| url | VARCHAR(512) NULL | 原文链接(新闻联播为空) | +| created_at | DATETIME | 入库时间 | + +索引:`idx_report_section (report_id, section)`。 + +--- + +## 2. 数据契约 + +- **幂等语义**:同一 `(report_date, report_type, file_name)` 重复写入会覆盖主表并全量替换事件(DELETE + INSERT),不会产生重复行。 +- **取最新**:同一天存在多份时(历史 intl 一日 3 次),前端按 `generated_at` 取最新;新日报 `file_name=''` 每天唯一。 +- **板块差异**:finance 日报含 `xwlb`+`news`+`cninfo` 三板块;intl 日报仅 `intl` 板块。前端按 `section` 过滤展示。 +- **历史覆盖范围**:2026-06-16 ~ 2026-08-03,共 177 行(finance 49 + intl 128;finance 少 1 因为两个目录存在同名文件被幂等合并)。事件总计 4222 条。 + +--- + +## 3. stats JSON 结构 + +`news_report.stats` 为数据总览快照,前端自行解析。finance 与 intl 的 key 集合不同: + +| key | finance | intl | 内容 | +| --- | --- | --- | --- | +| `pipeline` | ✅ | ✅ | M1→M6 管道各环节数量:`{label: 数量}` | +| `sources` | ✅ | — | 各新闻源文章数:`{源名: 数量}` | +| `news` | ✅ | — | 新闻统计:`{total, hi_threshold, sentiments, importances, event_types}` | +| `cninfo` | ✅ | — | 公告调研统计:`{total, hi_threshold, by_day, announcement, research, irm}` | +| `xwlb` | ✅ | — | 联播统计:`{total, date}`(有数据时才有) | +| `sentiment` | ✅ | ✅ | 情绪分布(历史文件为图例文本列表;新生成在 `news.sentiments`) | +| `importance` | ✅ | ✅ | 重要度分布:`[{重要度, 数量}, ...]` | +| `event_types` | ✅ | ✅ | 事件类型 TOP:`[{事件类型, 数量}, ...]` | +| `source_dist` | — | ✅ | 文章来源分布:`[{来源, 文章数}, ...]` | + +> 历史文件与新生成日报的 stats 结构存在差异(历史为 HTML 解析快照,新生成为结构化组装),前端建议按 key 防御性读取。 + +--- + +## 4. 常用查询示例(API 实现参考) + +```sql +-- 某类型日报列表(取每天最新一份) +SELECT r.* FROM news_report r +JOIN ( + SELECT report_date, report_type, MAX(generated_at) AS g + FROM news_report GROUP BY report_date, report_type +) t ON r.report_date = t.report_date AND r.report_type = t.report_type + AND r.generated_at = t.g +WHERE r.report_type = 'finance' AND r.report_date >= '2026-07-01' +ORDER BY r.report_date DESC; + +-- 某日报的全部事件(按板块) +SELECT section, rank, importance, event_type, title, summary, sentiment, source, url +FROM news_event WHERE report_id = ? ORDER BY section, rank; + +-- 最近 N 天重要事件聚合(跨日报检索) +SELECT e.* FROM news_event e +JOIN news_report r ON r.id = e.report_id +WHERE r.report_date >= DATE_SUB(CURDATE(), INTERVAL 7 DAY) + AND e.importance >= 4 +ORDER BY e.importance DESC, r.report_date DESC; +``` + +--- + +## 5. 相关命令(数据生产侧) + +```bash +uv run a-share report --date YYYYMMDD # 生成当日日报并入库(finance) +uv run a-share report-import # 历史 HTML 全量解析入库(幂等) +uv run a-share report-import --date YYYYMMDD --type intl +``` + +代码:`report_db/`(连接/写入)、`report_import/`(历史解析/导入)、`scheduler/reporter.py`(日报生成)。 diff --git a/djapi/docs/report_db_design.md b/djapi/docs/report_db_design.md new file mode 100644 index 0000000..3626b34 --- /dev/null +++ b/djapi/docs/report_db_design.md @@ -0,0 +1,330 @@ +# Milestone 10 后端实现逻辑:日报结构化入库 + +> 版本:v0.1(设计稿) | 2026-07 +> 对应 project_plan.md「十八、Milestone 10」 +> **范围**:本项目侧"后端"= 数据生产层(日报内容生成 + 结构化写入 MySQL)。 +> 不包含 API 服务与前端页面(由用户另行实现),但表结构与数据契约以本文档为准,供 API/前端对接。 + +--- + +## 1. 定位 + +现有链路:`reporter.py` 收集数据 → `_render_html()` 渲染 HTML → scp 上传 doorcome。 +改造后:`reporter.py` 收集数据 → 组装结构化 `ReportData` → 写入 MySQL(`news_report` / `news_event`),不再产出 HTML。 + +另需:把 doorcome 上 178 份历史日报 HTML(`finance_news_daily_*` ×50、`intl_news_daily_*` ×128)解析成同一 `ReportData` 结构入库。 + +--- + +## 2. 数据流总览 + +``` +[历史 HTML ×178] [每日 pipeline] + doorcome:/var/www/html/echart/research/ crawler→extractor→dedup→llm→embed→qdrant + (一次性 scp 到 data/reports_history/) │ + │ ▼ + ▼ reporter.generate_report() + report_import/parser.py │ + │ (BeautifulSoup 解析) ▼ + ▼ 组装 ReportData 组装 ReportData + report_import/importer.py │ + │ (幂等 upsert) ▼ + ▼ │ + ┌────────────────────── MySQL (myquant 库) ──────────────────────┐ + │ news_report(主表) news_event(事件明细) │ + └────────────────────────────────────────────────────────────────┘ + ▲ + API / 前端(用户另行实现,只读) +``` + +--- + +## 3. 数据模型(Pydantic,`report_db/models.py`) + +```python +class EventRow(BaseModel): + """一条事件记录,对应 news_event 一行。""" + section: str # xwlb | news | cninfo | intl + rank: int # 板块内序号(从 1 开始) + importance: int | None = None + event_type: str | None = None + title: str + summary: str | None = None + sentiment: str | None = None # positive | negative | neutral | '' + source: str | None = None # 来源(如 cls / ForexLive) + url: str | None = None + +class ReportData(BaseModel): + """一份完整日报,对应 news_report 一行 + news_event 多行。""" + report_date: date # 日报日期(YYYY-MM-DD) + report_type: str # finance | intl + file_name: str # 源文件名(新生成时可为 "") + generated_at: datetime # 生成时间 + ai_summary: str | None = None + stats: dict[str, Any] = Field(default_factory=dict) # 数据总览统计快照 → JSON 列 + events: list[EventRow] = Field(default_factory=list) +``` + +--- + +## 4. 字段映射(核心契约) + +### 4.1 事件 JSON(data/events/)→ news_event + +现有事件文件结构与 news_event 字段对应关系(reporter 收集时直接转换): + +| news_event 字段 | 事件 JSON 来源 | +| --- | --- | +| section | 来源判定:`source_id=="cninfo"` → `cninfo`;`source_id=="xwlb"` → `xwlb`;否则 `news`;intl 解析固定 `intl` | +| importance | `event.importance` | +| event_type | `event.event_type` | +| title | `title` | +| summary | `event.summary` | +| sentiment | `event.sentiment` | +| source | `source_id` | +| url | `url`(xwlb 为空) | + +### 4.2 历史 HTML → ReportData + +解析策略:**表头驱动列映射**。不同日报表格列集合不同: + +| 板块 | 表格列(