Initial commit: cc-cursor 全链路量化研究平台
7 Sprints 全部完成: Sprint 0: 基础设施 (DataManager + MariaDB) Sprint 1: 因子引擎 (34因子/12分类) Sprint 2: VectorBT 回测 (5策略+截面) Sprint 3: Optuna 优化 (+Walk-Forward) Sprint 4: ML 模型 (LightGBM+CatBoost) Sprint 5: Qwen 情绪因子 (三源新闻+日期对齐) Sprint 6: Agent 系统 (4Agent+日报.md/.html) 生产加固 (15项): Tushare双源fallback, SSH自动恢复, pool_pre_ping, save_daily先删后插, load_dotenv绝对路径, 日报5d/20d修复, RiskAgent改上证指数, 昨日对比+数据截止, mac_report utf8mb4, CLAUDE-*.md 9条已知Bug, demo全参数化, djapi数据源归一化, indexDatas API修正 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
@@ -0,0 +1,12 @@
|
||||
#api/__init__.py
|
||||
from . import stock
|
||||
|
||||
__version__ = "0.1.0"
|
||||
__all__ = ["stock"]
|
||||
|
||||
# 初始化代码
|
||||
def init():
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
init()
|
||||
@@ -0,0 +1,3 @@
|
||||
from django.contrib import admin
|
||||
|
||||
# Register your models here.
|
||||
@@ -0,0 +1,6 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ApiConfig(AppConfig):
|
||||
default_auto_field = 'django.db.models.BigAutoField'
|
||||
name = 'api'
|
||||
@@ -0,0 +1,22 @@
|
||||
from django.db import models
|
||||
|
||||
# Create your models here.
|
||||
class Book(models.Model):
|
||||
title = models.CharField(max_length=100)
|
||||
author = models.CharField(max_length=100)
|
||||
published_date = models.DateField()
|
||||
isbn_number = models.CharField(max_length=13)
|
||||
|
||||
def __str__(self):
|
||||
return self.title
|
||||
|
||||
# admin.py
|
||||
from django.contrib import admin
|
||||
from .models import Book
|
||||
|
||||
class BookAdmin(admin.ModelAdmin):
|
||||
list_display = ('title', 'author', 'published_date')
|
||||
search_fields = ('title', 'author')
|
||||
list_filter = ('published_date',)
|
||||
|
||||
admin.site.register(Book, BookAdmin)
|
||||
@@ -0,0 +1,145 @@
|
||||
from rest_framework import serializers
|
||||
|
||||
|
||||
class StockDailySerializer(serializers.Serializer):
|
||||
"""日线行情（stockbasic）"""
|
||||
ts_code = serializers.CharField()
|
||||
trade_date = serializers.CharField()
|
||||
open = serializers.FloatField()
|
||||
high = serializers.FloatField()
|
||||
low = serializers.FloatField()
|
||||
close = serializers.FloatField()
|
||||
pre_close = serializers.FloatField()
|
||||
change = serializers.FloatField()
|
||||
pct_chg = serializers.FloatField()
|
||||
vol = serializers.FloatField()
|
||||
amount = serializers.FloatField()
|
||||
|
||||
|
||||
class StockInfoSerializer(serializers.Serializer):
|
||||
"""个股基本信息（stockinfo）"""
|
||||
ts_code = serializers.CharField()
|
||||
symbol = serializers.CharField()
|
||||
name = serializers.CharField()
|
||||
area = serializers.CharField()
|
||||
industry = serializers.CharField()
|
||||
market = serializers.CharField()
|
||||
list_date = serializers.CharField()
|
||||
fullname = serializers.CharField()
|
||||
enname = serializers.CharField()
|
||||
exchange = serializers.CharField()
|
||||
curr_type = serializers.CharField()
|
||||
list_status = serializers.CharField()
|
||||
is_hs = serializers.CharField()
|
||||
|
||||
|
||||
class IndustryStockSerializer(serializers.Serializer):
|
||||
"""行业股票列表（industrys）"""
|
||||
ts_code = serializers.CharField()
|
||||
name = serializers.CharField()
|
||||
|
||||
|
||||
class StockParamSerializer(serializers.Serializer):
|
||||
"""个股参数（stockparam）"""
|
||||
ts_code = serializers.CharField()
|
||||
trade_date = serializers.CharField()
|
||||
close = serializers.FloatField()
|
||||
turnover_rate = serializers.FloatField()
|
||||
turnover_rate_f = serializers.FloatField()
|
||||
volume_ratio = serializers.FloatField()
|
||||
pe = serializers.FloatField()
|
||||
pe_ttm = serializers.FloatField()
|
||||
pb = serializers.FloatField()
|
||||
ps = serializers.FloatField()
|
||||
ps_ttm = serializers.FloatField()
|
||||
dv_ratio = serializers.FloatField()
|
||||
dv_ttm = serializers.FloatField()
|
||||
total_share = serializers.FloatField()
|
||||
float_share = serializers.FloatField()
|
||||
free_share = serializers.FloatField()
|
||||
total_mv = serializers.FloatField()
|
||||
circ_mv = serializers.FloatField()
|
||||
|
||||
|
||||
class StockEpSerializer(serializers.Serializer):
|
||||
"""TTM EPS（stockep）"""
|
||||
ts_code = serializers.CharField()
|
||||
trade_date = serializers.CharField()
|
||||
eps_ttm = serializers.FloatField()
|
||||
|
||||
|
||||
class QuarterlyEpsSerializer(serializers.Serializer):
|
||||
"""季度 EPS（quarterlyEps）"""
|
||||
ts_code = serializers.CharField()
|
||||
trade_date = serializers.CharField()
|
||||
eps = serializers.FloatField()
|
||||
report_date = serializers.CharField()
|
||||
|
||||
|
||||
class IndexInfoSerializer(serializers.Serializer):
|
||||
"""指数信息（indexByName）"""
|
||||
index_code = serializers.CharField()
|
||||
name = serializers.CharField()
|
||||
fullname = serializers.CharField()
|
||||
market = serializers.CharField()
|
||||
publisher = serializers.CharField()
|
||||
index_type = serializers.CharField()
|
||||
category = serializers.CharField()
|
||||
list_date = serializers.CharField()
|
||||
|
||||
|
||||
class IndexDailySerializer(serializers.Serializer):
|
||||
"""指数日行情（indexDatas）"""
|
||||
ts_code = serializers.CharField()
|
||||
trade_date = serializers.CharField()
|
||||
close = serializers.FloatField()
|
||||
open = serializers.FloatField()
|
||||
high = serializers.FloatField()
|
||||
low = serializers.FloatField()
|
||||
pre_close = serializers.FloatField()
|
||||
change = serializers.FloatField()
|
||||
pct_chg = serializers.FloatField()
|
||||
vol = serializers.FloatField()
|
||||
amount = serializers.FloatField()
|
||||
|
||||
|
||||
class MarginDailySerializer(serializers.Serializer):
|
||||
"""每日融资融券汇总（dailyMargin）"""
|
||||
trade_date = serializers.CharField()
|
||||
exchange_id = serializers.CharField()
|
||||
rzye = serializers.FloatField()
|
||||
rqye = serializers.FloatField()
|
||||
rzrqye = serializers.FloatField()
|
||||
|
||||
|
||||
class StockMarginSerializer(serializers.Serializer):
|
||||
"""个股融资融券（stockMargin）"""
|
||||
ts_code = serializers.CharField()
|
||||
trade_date = serializers.CharField()
|
||||
rzye = serializers.FloatField()
|
||||
rqye = serializers.FloatField()
|
||||
rzrqye = serializers.FloatField()
|
||||
|
||||
|
||||
class FinanceDataSerializer(serializers.Serializer):
|
||||
"""财务报表分析（finance）"""
|
||||
ts_code = serializers.CharField()
|
||||
period = serializers.CharField()
|
||||
|
||||
|
||||
class DividendSerializer(serializers.Serializer):
|
||||
"""股息率（getdiv）"""
|
||||
ts_code = serializers.CharField()
|
||||
trade_date = serializers.CharField()
|
||||
close = serializers.FloatField()
|
||||
cash_div_tax = serializers.FloatField()
|
||||
cash_div_year = serializers.FloatField()
|
||||
div_yield = serializers.FloatField()
|
||||
|
||||
|
||||
class XwlbNewsSerializer(serializers.Serializer):
|
||||
"""新闻联播（xwlbNews / xwlbFine）"""
|
||||
news_days = serializers.CharField()
|
||||
daily_sub_id = serializers.IntegerField()
|
||||
news_improve = serializers.CharField()
|
||||
news_title = serializers.CharField()
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,9 @@
|
||||
# api/stock/__init__.py
|
||||
from . import config
|
||||
from . import getIndexs
|
||||
from . import getStockEp
|
||||
from . import getStockParam
|
||||
from . import stock_basic
|
||||
from . import stock_utils
|
||||
|
||||
__all__ = ["config","getIndexs","getStockEp","getStockParam","stock_basic","stock_utils"]
|
||||
@@ -0,0 +1,21 @@
|
||||
import os
|
||||
|
||||
# Tushare API Token（来自环境变量）
|
||||
TS_TOKEN = os.getenv('TUSHARE_TS_TOKEN', '')
|
||||
|
||||
# 股票代码
|
||||
TS_CODE = '002273.SZ'
|
||||
|
||||
# 数据日期范围
|
||||
START_DATE = '20200101'
|
||||
END_DATE = '20251231'
|
||||
|
||||
# 常规配置
|
||||
PRECISION_CONFIG = 4 #小数点后精度默认配置
|
||||
|
||||
# 策略参数 — 拆分自 strategy_config
|
||||
from .strategy_config import * # noqa: F401, F403
|
||||
|
||||
# 扫描配置 — 拆分自 scan_config
|
||||
from .scan_config import * # noqa: F401, F403
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""
|
||||
统一数据源模块 — djapi/api/stock/ 的单一数据入口。
|
||||
|
||||
归一化 Tushare / AkShare / MySQL 三种数据源。
|
||||
所有模块通过此入口获取数据连接，不再各自创建 pro 实例。
|
||||
|
||||
用法:
|
||||
from .data_source import get_tushare_pro, get_mysql_db
|
||||
|
||||
pro = get_tushare_pro()
|
||||
df = pro.daily(ts_code='000001.SZ', ...)
|
||||
|
||||
db = get_mysql_db()
|
||||
rows = db.query("SELECT * FROM xwlb_daily WHERE ...")
|
||||
|
||||
扩展新数据源:
|
||||
class NewSource: ...
|
||||
_sources['new'] = NewSource()
|
||||
def get_new_source(): return _sources['new']
|
||||
"""
|
||||
|
||||
import os
|
||||
import threading
|
||||
|
||||
import tushare as ts
|
||||
|
||||
# 延迟加载 AkShare（避免不必要的导入开销）
|
||||
_akshare = None
|
||||
|
||||
|
||||
def _get_akshare():
|
||||
global _akshare
|
||||
if _akshare is None:
|
||||
import akshare as ak
|
||||
_akshare = ak
|
||||
return _akshare
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# Token 加载
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def get_ts_token() -> str:
|
||||
"""获取 Tushare Token，优先级：环境变量 TUSHARE_TS_TOKEN > TUSHARE_TOKEN > config.py"""
|
||||
token = os.getenv("TUSHARE_TS_TOKEN", "") or os.getenv("TUSHARE_TOKEN", "")
|
||||
if not token:
|
||||
try:
|
||||
from .config import TS_TOKEN
|
||||
token = TS_TOKEN
|
||||
except ImportError:
|
||||
try:
|
||||
from config import TS_TOKEN
|
||||
token = TS_TOKEN
|
||||
except ImportError:
|
||||
pass
|
||||
return token
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# Tushare Pro 连接池（线程安全单例）
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
_pro_lock = threading.Lock()
|
||||
_pro = None
|
||||
|
||||
|
||||
def get_tushare_pro():
|
||||
"""获取 Tushare pro_api 实例（全局单例，线程安全）。"""
|
||||
global _pro
|
||||
if _pro is not None:
|
||||
return _pro
|
||||
with _pro_lock:
|
||||
if _pro is not None:
|
||||
return _pro
|
||||
token = get_ts_token()
|
||||
ts.set_token(token)
|
||||
_pro = ts.pro_api()
|
||||
return _pro
|
||||
|
||||
|
||||
def reset_tushare_pro():
|
||||
"""重置 Tushare 连接（token 变更时调用）。"""
|
||||
global _pro
|
||||
with _pro_lock:
|
||||
_pro = None
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# MySQL 连接
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
_mysql_db = None
|
||||
|
||||
|
||||
def get_mysql_db():
|
||||
"""获取 MySQLDB 实例（全局单例）。"""
|
||||
global _mysql_db
|
||||
if _mysql_db is not None:
|
||||
return _mysql_db
|
||||
try:
|
||||
from ..utils.mysql_handler import MySQLDB
|
||||
except (ImportError, ValueError):
|
||||
try:
|
||||
from utils.mysql_handler import MySQLDB
|
||||
except ImportError:
|
||||
return None
|
||||
_mysql_db = MySQLDB()
|
||||
return _mysql_db
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 日线行情 — 双源 fallback（Tushare → AkShare）
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
|
||||
def get_daily(ts_code: str, start_date: str, end_date: str, source: str = "tushare"):
|
||||
"""
|
||||
获取个股日线行情。
|
||||
|
||||
参数:
|
||||
ts_code: 如 '000001.SZ'
|
||||
start_date: YYYYMMDD
|
||||
end_date: YYYYMMDD
|
||||
source: 'tushare' | 'akshare' | 'auto' (tushare优先)
|
||||
|
||||
返回:
|
||||
pd.DataFrame (trade_date, open, high, low, close, vol, amount, ...)
|
||||
"""
|
||||
import pandas as pd
|
||||
|
||||
if source == "auto":
|
||||
# Tushare 优先
|
||||
try:
|
||||
df = get_daily(ts_code, start_date, end_date, source="tushare")
|
||||
if df is not None and not df.empty:
|
||||
return df
|
||||
except Exception:
|
||||
pass
|
||||
return get_daily(ts_code, start_date, end_date, source="akshare")
|
||||
|
||||
if source == "tushare":
|
||||
pro = get_tushare_pro()
|
||||
df = pro.daily(
|
||||
ts_code=ts_code, start_date=start_date, end_date=end_date,
|
||||
fields="ts_code,trade_date,open,high,low,close,pre_close,change,pct_chg,vol,amount"
|
||||
)
|
||||
if df is not None and not df.empty:
|
||||
df["trade_date"] = df["trade_date"].astype(str)
|
||||
return df
|
||||
|
||||
if source == "akshare":
|
||||
symbol = ts_code.replace(".SZ", "").replace(".SH", "").replace(".BJ", "")
|
||||
ak = _get_akshare()
|
||||
df = ak.stock_zh_a_hist(
|
||||
symbol=symbol, period="daily",
|
||||
start_date=start_date, end_date=end_date, adjust="qfq"
|
||||
)
|
||||
if df is not None and not df.empty:
|
||||
df = df.rename(columns={
|
||||
"日期": "trade_date", "开盘": "open", "收盘": "close",
|
||||
"最高": "high", "最低": "low", "成交量": "vol", "成交额": "amount",
|
||||
"涨跌幅": "pct_chg", "涨跌额": "change",
|
||||
})
|
||||
df["ts_code"] = ts_code
|
||||
df["trade_date"] = df["trade_date"].astype(str)
|
||||
return df if df is not None else pd.DataFrame()
|
||||
|
||||
raise ValueError("Unknown source: {}".format(source))
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
# 扩展点：未来新增数据源
|
||||
# ═══════════════════════════════════════════════════════════
|
||||
#
|
||||
# 1. 在 _sources dict 中注册新源
|
||||
# 2. 实现与 get_daily() 相同签名的函数
|
||||
# 3. 在 get_daily(source=...) 中添加路由
|
||||
#
|
||||
# _sources = {
|
||||
# "tushare": TushareDailySource(),
|
||||
# "akshare": AkShareDailySource(),
|
||||
# "wind": WindDailySource(), # 未来
|
||||
# "joinquant": JoinQuantSource(), # 未来
|
||||
# }
|
||||
@@ -0,0 +1,219 @@
|
||||
import pandas as pd
|
||||
|
||||
from config import START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from stock_utils import *
|
||||
from getStockDiv2 import analyze_stock_dividend_and_price
|
||||
from getStockParam import getStockParam
|
||||
from .data_source import get_tushare_pro
|
||||
import datetime
|
||||
|
||||
pro = get_tushare_pro()
|
||||
|
||||
# --- 数据输出配置 ---
|
||||
OUTPUT_TO_CSV = True # 开关：是否输出到CSV
|
||||
OUTPUT_TO_MYSQL = False # 开关：是否输出到MySQL (需要额外配置数据库连接)
|
||||
|
||||
|
||||
# --- 数据库输出函数 (伪代码) ---
|
||||
def save_to_mysql(dataframe, table_name):
|
||||
"""
|
||||
将DataFrame保存到MySQL数据库的伪代码函数。
|
||||
实际使用时需要配置数据库连接。
|
||||
"""
|
||||
if not OUTPUT_TO_MYSQL:
|
||||
print("MySQL输出已禁用。")
|
||||
return
|
||||
|
||||
print(f"正在将数据保存到MySQL表 {table_name}...")
|
||||
# --- 伪代码开始 ---
|
||||
# import pymysql # 或其他数据库连接库
|
||||
#
|
||||
# connection = pymysql.connect(host='your_host',
|
||||
# user='your_user',
|
||||
# password='your_password',
|
||||
# database='your_database',
|
||||
# charset='utf8mb4')
|
||||
# try:
|
||||
# with connection.cursor() as cursor:
|
||||
# # 创建表 (如果不存在)
|
||||
# # create_table_sql = "..."
|
||||
# # cursor.execute(create_table_sql)
|
||||
#
|
||||
# # 遍历DataFrame并插入数据
|
||||
# for index, row in dataframe.iterrows():
|
||||
# # 注意：需要处理SQL注入风险，最好使用参数化查询
|
||||
# insert_sql = f"INSERT INTO {table_name} (...) VALUES (...)"
|
||||
# cursor.execute(insert_sql, tuple(row))
|
||||
# connection.commit()
|
||||
# print(f"成功保存 {len(dataframe)} 条记录到MySQL。")
|
||||
# except Exception as e:
|
||||
# print(f"保存到MySQL时出错: {e}")
|
||||
# connection.rollback()
|
||||
# finally:
|
||||
# connection.close()
|
||||
# --- 伪代码结束 ---
|
||||
print("MySQL保存操作完成 (伪代码)。")
|
||||
|
||||
|
||||
# --- CSV输出函数 ---
|
||||
def save_to_csv(dataframe, filename, mode='w', header=True):
|
||||
"""将DataFrame保存到CSV文件"""
|
||||
if not OUTPUT_TO_CSV:
|
||||
print("CSV输出已禁用。")
|
||||
return
|
||||
|
||||
try:
|
||||
dataframe.to_csv(filename, index=False, encoding='utf-8', mode=mode, header=header)
|
||||
print(f"成功保存 {len(dataframe)} 条记录到 {filename}")
|
||||
except Exception as e:
|
||||
print(f"保存CSV文件 {filename} 时出错: {e}")
|
||||
|
||||
'''
|
||||
写一个函数，调用上面的函数，传入股票代码和起止日期，计算div_yield的最大值，最小值，均值，最新值，以及标准差，返回pandas的Series
|
||||
'''
|
||||
def calculate_dividend_yield_stats(ts_code, start_date=START_DATE, end_date=END_DATE):
|
||||
"""
|
||||
计算股息率的统计指标
|
||||
|
||||
参数:
|
||||
ts_code (str): 股票代码
|
||||
start_date (str): 起始日期
|
||||
end_date (str): 结束日期
|
||||
|
||||
返回:
|
||||
pd.Series: 包含股息率统计指标的Series
|
||||
"""
|
||||
df = analyze_stock_dividend_and_price(ts_code, start_date, end_date)
|
||||
|
||||
if df.empty or 'div_yield' not in df.columns:
|
||||
return pd.Series(dtype=float)
|
||||
|
||||
div_yield_data = df['div_yield']
|
||||
|
||||
stats = pd.Series({
|
||||
'ts_code': ts_code,
|
||||
'max': div_yield_data.max(),
|
||||
'min': div_yield_data.min(),
|
||||
'mean': div_yield_data.mean(),
|
||||
'latest': div_yield_data.iloc[-1] if len(div_yield_data) > 0 else 0,
|
||||
'std': div_yield_data.std(),
|
||||
'zero_count': (div_yield_data == 0).sum()
|
||||
})
|
||||
|
||||
return stats
|
||||
|
||||
# --- 主处理函数 ---
|
||||
def process_stock_dividend_batch(start_dt='20200101', end_dt='20251231', dataInterval=10, exchange='SSE'):
|
||||
"""
|
||||
批量处理股票股息率数据，并分批输出。
|
||||
"""
|
||||
print(f"开始处理 {exchange} 交易所的股票数据，时间范围: {start_dt} - {end_dt}")
|
||||
|
||||
try:
|
||||
df_stocks = get_stock_basic(exchange=exchange)
|
||||
if df_stocks.empty:
|
||||
print("未获取到股票基础数据")
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"获取股票基础数据时出错: {e}")
|
||||
return
|
||||
|
||||
results_batch = []
|
||||
all_results = []
|
||||
processed_count = 0
|
||||
total_stocks = len(df_stocks)
|
||||
csv_filename = f"{exchange}_dividend_yield_stats_batch.csv"
|
||||
csv_first_write = True # 用于控制CSV文件头只写入一次
|
||||
|
||||
for idx, row in df_stocks.iterrows():
|
||||
stock_code = row['ts_code']
|
||||
print(f"正在处理 {stock_code} ({idx+1}/{total_stocks})")
|
||||
|
||||
try:
|
||||
raw_stats = calculate_dividend_yield_stats(stock_code, start_date=start_dt, end_date=end_dt)
|
||||
|
||||
# 确保我们最终得到一个 Series (统计数据)
|
||||
stats_series = None
|
||||
if isinstance(raw_stats, pd.Series):
|
||||
# 如果直接返回了 Series
|
||||
stats_series = raw_stats
|
||||
elif isinstance(raw_stats, pd.DataFrame) and not raw_stats.empty:
|
||||
# 如果返回了 DataFrame，则取第一行
|
||||
stats_series = raw_stats.iloc[0]
|
||||
else:
|
||||
# 如果返回了空DataFrame, None, 或其他类型
|
||||
print(f"股票 {stock_code} 的 calculate_dividend_yield_stats 返回了空数据或无效类型 ({type(raw_stats)})。跳过...")
|
||||
continue
|
||||
|
||||
|
||||
yesterday = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y%m%d")
|
||||
stockParam = getStockParam(stock_code, START_DATE=yesterday, END_DATE=yesterday)
|
||||
# 选择需要的字段与下方合并
|
||||
if not stockParam.empty:
|
||||
selected_params_dict = {
|
||||
'ts_code': stockParam['ts_code'].iloc[0],
|
||||
'trade_date': stockParam['trade_date'].iloc[0],
|
||||
'total_mv': stockParam['total_mv'].iloc[0],
|
||||
'circ_mv': stockParam['circ_mv'].iloc[0]
|
||||
}
|
||||
else:
|
||||
selected_params_dict = { 'ts_code': stock_code, 'trade_date': None, 'total_mv': None,'circ_mv': None}
|
||||
|
||||
# --- 修改点2：简化合并逻辑 ---
|
||||
# 现在 stats_series 已确认是 Series，可以直接合并
|
||||
# 使用字典合并确保结果清晰且 stats 覆盖同名项
|
||||
combined_dict = {**row.to_dict(), **stats_series.to_dict(), **selected_params_dict}
|
||||
combined_series = pd.Series(combined_dict)
|
||||
print(f"合并结果: {combined_series}")
|
||||
print(f"{stock_code}-{start_dt} -- {end_dt} 处理完成。")
|
||||
results_batch.append(combined_series)
|
||||
all_results.append(combined_series)
|
||||
|
||||
except Exception as e:
|
||||
print(f"处理股票 {stock_code} 时出错: {e}")
|
||||
# 可以选择在这里添加错误记录到 results_batch 或 all_results
|
||||
continue
|
||||
|
||||
processed_count += 1
|
||||
|
||||
# 检查是否达到批次大小
|
||||
if processed_count % dataInterval == 0 and results_batch:
|
||||
print(f"已处理 {processed_count} 支股票，达到批次大小 {dataInterval}，开始输出...")
|
||||
df_batch = pd.DataFrame(results_batch)
|
||||
|
||||
# 输出到CSV (追加模式)
|
||||
save_to_csv(df_batch, csv_filename, mode='a', header=csv_first_write)
|
||||
if csv_first_write: csv_first_write = False # 之后的批次不再写入header
|
||||
|
||||
# 输出到MySQL (伪代码)
|
||||
save_to_mysql(df_batch, 'stock_dividend_stats')
|
||||
|
||||
# 清空批次缓存
|
||||
results_batch = []
|
||||
print("--- 批次处理完成 ---")
|
||||
|
||||
# 处理最后一批不足dataInterval的数据
|
||||
if results_batch:
|
||||
print(f"处理剩余 {len(results_batch)} 支股票...")
|
||||
df_final_batch = pd.DataFrame(results_batch)
|
||||
save_to_csv(df_final_batch, csv_filename, mode='a', header=csv_first_write)
|
||||
save_to_mysql(df_final_batch, 'stock_dividend_stats')
|
||||
print("--- 最终批次处理完成 ---")
|
||||
|
||||
# (可选) 将所有结果一次性保存到一个完整的CSV文件
|
||||
if all_results:
|
||||
final_csv_filename = f"{exchange}_dividend_yield_stats_final.csv"
|
||||
print(f"正在保存所有 {len(all_results)} 条结果到 {final_csv_filename}...")
|
||||
df_final = pd.DataFrame(all_results)
|
||||
save_to_csv(df_final, final_csv_filename, mode='w', header=True)
|
||||
save_to_mysql(df_final, 'stock_dividend_stats_final')
|
||||
print("所有数据处理并保存完成。")
|
||||
else:
|
||||
print("无有效结果数据可保存")
|
||||
|
||||
# --- 程序入口 ---
|
||||
if __name__ == "__main__":
|
||||
# 可以通过修改这里的参数来调用函数
|
||||
process_stock_dividend_batch(start_dt='20150101', end_dt='20251231', dataInterval=10, exchange='SSE')
|
||||
|
||||
process_stock_dividend_batch(start_dt='20150101', end_dt='20251231', dataInterval=10, exchange='SZSE')
|
||||
@@ -0,0 +1,181 @@
|
||||
"""
|
||||
⚠️ 已废弃 — 2026-06-05
|
||||
|
||||
AkShare 股息率数据获取功能已归一化到 data_source.py。
|
||||
如需 AkShare 日线数据，请使用：
|
||||
|
||||
from .data_source import get_daily
|
||||
df = get_daily(ts_code, start, end, source='akshare')
|
||||
|
||||
本文件保留仅用于向后兼容，所有公开函数委托给统一数据源。
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from .data_source import get_daily
|
||||
|
||||
from .stock_utils import tscodeCheck, date_format_correction
|
||||
from .smoothBrush import smooth_dataframe_brush
|
||||
|
||||
GAP_DAYS = 360 # TTM 计算窗口
|
||||
|
||||
|
||||
def _parse_tscode(tscode: str) -> tuple:
|
||||
"""将 tscode（如 000001.SZ）转为 akshare 格式的 symbol 和 market"""
|
||||
code = tscode.split('.')[0]
|
||||
suffix = tscode.split('.')[1].lower()
|
||||
market_map = {'sz': 'sz', 'sh': 'sh', 'bj': 'bj'}
|
||||
return code, market_map.get(suffix, suffix)
|
||||
|
||||
|
||||
def _fetch_daily_price(symbol: str, start_date: str, end_date: str) -> pd.DataFrame:
|
||||
"""通过 akshare 获取前复权日线行情"""
|
||||
df = ak.stock_zh_a_hist(
|
||||
symbol=symbol,
|
||||
period='daily',
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
adjust='qfq'
|
||||
)
|
||||
if df.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
df = df.rename(columns={
|
||||
'日期': 'trade_date',
|
||||
'开盘': 'open',
|
||||
'收盘': 'close',
|
||||
'最高': 'high',
|
||||
'最低': 'low',
|
||||
'成交量': 'vol',
|
||||
'成交额': 'amount',
|
||||
'换手率': 'turnover_rate',
|
||||
})
|
||||
df['trade_date'] = df['trade_date'].astype(str).str.replace('-', '')
|
||||
return df
|
||||
|
||||
|
||||
def _fetch_dividends(symbol: str, market: str) -> pd.DataFrame:
|
||||
"""通过 akshare 获取历史分红记录"""
|
||||
try:
|
||||
df = ak.stock_dividend_cninfo(stock=symbol, symbol=market + symbol)
|
||||
except Exception:
|
||||
return pd.DataFrame()
|
||||
|
||||
if df.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
# 列名映射（akshare 返回中文列名）
|
||||
col_map = {
|
||||
'除权除息日': 'ex_date',
|
||||
'每股派息': 'cash_div_tax',
|
||||
}
|
||||
df = df.rename(columns={k: v for k, v in col_map.items() if k in df.columns})
|
||||
|
||||
if 'ex_date' not in df.columns or 'cash_div_tax' not in df.columns:
|
||||
return pd.DataFrame()
|
||||
|
||||
df['ex_date'] = df['ex_date'].astype(str).str.replace('-', '').str[:8]
|
||||
df['cash_div_tax'] = pd.to_numeric(df['cash_div_tax'], errors='coerce').fillna(0)
|
||||
return df[['ex_date', 'cash_div_tax']]
|
||||
|
||||
|
||||
def get_akshare_dividend_yield(tscode: str, start_date: str = None, end_date: str = None) -> pd.DataFrame:
|
||||
"""
|
||||
通过 akshare 获取股价和分红数据，计算股息率。
|
||||
|
||||
Args:
|
||||
tscode: 股票代码，如 '000001.SZ'
|
||||
start_date: 起始日期 yyyyMMdd
|
||||
end_date: 结束日期 yyyyMMdd
|
||||
|
||||
Returns:
|
||||
DataFrame: ts_code, trade_date, close, cash_div_tax, cash_div_year, div_yield
|
||||
"""
|
||||
tscode = tscodeCheck(tscode)
|
||||
today = datetime.now().strftime('%Y%m%d')
|
||||
|
||||
if start_date:
|
||||
start_date = date_format_correction(start_date)
|
||||
else:
|
||||
start_date = '20200101'
|
||||
if end_date:
|
||||
end_date = date_format_correction(end_date)
|
||||
else:
|
||||
end_date = today
|
||||
|
||||
if end_date > today:
|
||||
end_date = today
|
||||
|
||||
symbol, market = _parse_tscode(tscode)
|
||||
|
||||
# 1. 获取日线行情（运算时起始日期往前推 GAP_DAYS）
|
||||
calc_start = datetime.strptime(start_date, '%Y%m%d') - timedelta(days=GAP_DAYS)
|
||||
calc_start_str = calc_start.strftime('%Y%m%d')
|
||||
|
||||
df_price = _fetch_daily_price(symbol, calc_start_str, end_date)
|
||||
if df_price.empty:
|
||||
return pd.DataFrame()
|
||||
df_price = df_price.sort_values('trade_date').reset_index(drop=True)
|
||||
|
||||
# 2. 获取分红数据
|
||||
df_div = _fetch_dividends(symbol, market)
|
||||
|
||||
# 3. 计算 TTM 分红序列
|
||||
if df_div.empty:
|
||||
df_price['cash_div_tax'] = 0.0
|
||||
df_price['cash_div_year'] = 0.0
|
||||
else:
|
||||
# 将分红按 ex_date 合并到交易日历
|
||||
df_div['ex_date'] = pd.to_datetime(df_div['ex_date'], format='%Y%m%d')
|
||||
df_price['trade_date_dt'] = pd.to_datetime(df_price['trade_date'], format='%Y%m%d')
|
||||
|
||||
# 按日期合并
|
||||
div_dict = df_div.set_index('ex_date')['cash_div_tax'].to_dict()
|
||||
|
||||
def calc_ttm_div(trade_dt):
|
||||
window_start = trade_dt - timedelta(days=GAP_DAYS)
|
||||
total = 0.0
|
||||
for ex_dt, cash in div_dict.items():
|
||||
if window_start < ex_dt <= trade_dt:
|
||||
total += cash
|
||||
return total
|
||||
|
||||
cash_div_tax_list = []
|
||||
cash_div_year_list = []
|
||||
|
||||
for _, row in df_price.iterrows():
|
||||
td = row['trade_date_dt']
|
||||
cash = div_dict.get(td, 0.0)
|
||||
cash_div_tax_list.append(cash)
|
||||
cash_div_year_list.append(calc_ttm_div(td))
|
||||
|
||||
df_price['cash_div_tax'] = cash_div_tax_list
|
||||
df_price['cash_div_year'] = cash_div_year_list
|
||||
df_price = df_price.drop(columns=['trade_date_dt'])
|
||||
|
||||
# 4. 毛刺平滑
|
||||
df_price = smooth_dataframe_brush(
|
||||
df_price,
|
||||
target_columns=['cash_div_year'],
|
||||
window_size=31,
|
||||
threshold_factor=0.5,
|
||||
max_brush_length=15
|
||||
)
|
||||
|
||||
# 5. 截取请求的时间范围
|
||||
df_result = df_price[
|
||||
(df_price['trade_date'] >= start_date) & (df_price['trade_date'] <= end_date)
|
||||
].copy()
|
||||
|
||||
# 6. 计算股息率
|
||||
df_result['div_yield'] = 0.0
|
||||
mask = (df_result['close'] > 0) & (df_result['cash_div_year'] > 0)
|
||||
df_result.loc[mask, 'div_yield'] = (
|
||||
(df_result.loc[mask, 'cash_div_year'] / df_result.loc[mask, 'close']) * 100
|
||||
).round(4)
|
||||
|
||||
df_result['ts_code'] = tscode
|
||||
final_cols = ['ts_code', 'trade_date', 'close', 'cash_div_tax', 'cash_div_year', 'div_yield']
|
||||
return df_result[final_cols].reset_index(drop=True)
|
||||
@@ -0,0 +1,145 @@
|
||||
import pandas as pd
|
||||
|
||||
|
||||
# 导入当站目录的config文件
|
||||
try:
|
||||
# 尝试相对导入（作为包的一部分）
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from .stock_utils import *
|
||||
except (ImportError, SystemError):
|
||||
# 失败则使用绝对导入（直接运行脚本）
|
||||
from config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from stock_utils import *
|
||||
|
||||
from .data_source import get_tushare_pro
|
||||
pro = get_tushare_pro()
|
||||
|
||||
def get_index_by_name(index_name: str) -> pd.DataFrame:
|
||||
"""
|
||||
通过指数名称查询指数基本信息（使用tushare的index_basic接口）
|
||||
|
||||
Args:
|
||||
index_name (str): 要查询的指数名称（支持模糊匹配，如"上证"）
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含查询结果的DataFrame，列包括:
|
||||
- ts_code: 指数代码
|
||||
- name: 指数名称
|
||||
- fullname: 指数全称
|
||||
- market: 市场
|
||||
- publisher: 发布方
|
||||
- index_type: 指数类型
|
||||
- etc.
|
||||
|
||||
Raises:
|
||||
Exception: 当tushare接口调用失败时抛出异常
|
||||
|
||||
Example:
|
||||
>>> df = get_index_by_name("上证50")
|
||||
>>> print(df[['ts_code', 'name']])
|
||||
"""
|
||||
try:
|
||||
# 调用tushare接口查询指数信息
|
||||
df = pro.index_basic(name=index_name)
|
||||
|
||||
# 检查返回结果是否为空
|
||||
if df.empty:
|
||||
print(f"未找到名称包含'{index_name}'的指数")
|
||||
return pd.DataFrame() # 返回空DataFrame保持类型一致
|
||||
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
print(f"查询指数信息失败: {str(e)}")
|
||||
return pd.DataFrame()
|
||||
|
||||
def get_index_daily_data(ts_code: str, start_date: str = START_DATE, end_date: str = END_DATE) -> pd.DataFrame:
|
||||
"""
|
||||
通过指数代码查询日线行情数据（合并基本行情和扩展行情）
|
||||
|
||||
Args:
|
||||
ts_code (str): 指数代码（如"000001.SH"）
|
||||
start_date (str): 开始日期（格式"YYYYMMDD"，默认使用config中的START_DATE）
|
||||
end_date (str): 结束日期（格式"YYYYMMDD"，默认使用config中的END_DATE）
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 合并后的日线行情数据，包含以下列（示例）:
|
||||
- ts_code: 指数代码
|
||||
- trade_date: 交易日期
|
||||
- close: 收盘点位
|
||||
- open: 开盘点位
|
||||
- high: 最高点位
|
||||
- low: 最低点位
|
||||
- pe: 市盈率
|
||||
- pb: 市净率
|
||||
- etc.
|
||||
|
||||
Raises:
|
||||
Exception: 当tushare接口调用失败或数据合并失败时抛出异常
|
||||
|
||||
Example:
|
||||
>>> df = get_index_daily_data("000001.SH")
|
||||
>>> print(df[['trade_date', 'close', 'pe']].head())
|
||||
"""
|
||||
try:
|
||||
start_date = date_format_correction(start_date)
|
||||
end_date = date_format_correction(end_date)
|
||||
|
||||
# 1. 查询日线基本行情（核心数据，必须成功）
|
||||
daily_df = pro.index_daily(
|
||||
ts_code=ts_code,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
fields="ts_code,trade_date,open,high,low,close,pre_close,change,pct_chg,vol,amount"
|
||||
)
|
||||
if daily_df is None or daily_df.empty:
|
||||
print("未找到指数{}在{}到{}期间的行情数据".format(ts_code, start_date, end_date))
|
||||
return pd.DataFrame()
|
||||
|
||||
# 2. 尝试查询扩展行情（PE/PB 等，需要高权限，失败不阻塞）
|
||||
try:
|
||||
daily_basic_df = pro.index_dailybasic(
|
||||
ts_code=ts_code,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
fields="ts_code,trade_date,total_mv,float_mv,pe,pe_ttm,pb,turnover_rate,turnover_rate_f"
|
||||
)
|
||||
if daily_basic_df is not None and not daily_basic_df.empty:
|
||||
daily_df = pd.merge(
|
||||
daily_df, daily_basic_df,
|
||||
on=['ts_code', 'trade_date'],
|
||||
how='left'
|
||||
)
|
||||
except Exception as e:
|
||||
print("index_dailybasic 不可用（权限不足或接口变更）: {}".format(e))
|
||||
|
||||
return daily_df
|
||||
|
||||
except Exception as e:
|
||||
print("查询指数日线行情失败: {}".format(e))
|
||||
raise # 抛出而非静默返回空，让 view 层返回错误信息
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试指数查询
|
||||
print("测试指数查询:")
|
||||
index_df = get_index_by_name("中证500")
|
||||
if not index_df.empty:
|
||||
print(index_df[['ts_code', 'name']].head())
|
||||
else:
|
||||
print("未查询到指数信息")
|
||||
|
||||
'''# 测试指数行情获取
|
||||
print("\n测试指数日线行情:")
|
||||
test_code = "000016.SH" # 上证50指数代码
|
||||
daily_data = get_index_daily_data(test_code, start_date="20230101", end_date="20231231")
|
||||
|
||||
if not daily_data.empty:
|
||||
print(f"获取到{daily_data.shape[0]}条数据")
|
||||
# 显示关键列的前5行
|
||||
print(daily_data[['trade_date', 'close', 'pe', 'pb']].head())
|
||||
|
||||
# 数据完整性检查
|
||||
print("\n数据完整性检查:")
|
||||
print(daily_data[['close', 'pe']].describe())
|
||||
else:
|
||||
print(f"未获取到指数{test_code}的行情数据")'''
|
||||
@@ -0,0 +1,135 @@
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
# 尝试相对导入（作为包的一部分）
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from .stock_utils import *
|
||||
except (ImportError, SystemError):
|
||||
# 失败则使用绝对导入（直接运行脚本）
|
||||
from config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from stock_utils import *
|
||||
|
||||
'''
|
||||
给定个股代码和起止日期：
|
||||
1. 调用tushare接口查询个股分红数据，接口文档：https://tushare.pro/document/2?doc_id=103 查询日期范围内的所有分红记录
|
||||
2. 返回调整后的分红记录，包含以下字段：
|
||||
- ts_code: 股票代码
|
||||
- end_date: 分红年度
|
||||
- ann_date: 公告日期
|
||||
- ex_date: 除权除息日
|
||||
- cash_div_tax: 每股现金分红（含税）
|
||||
- div_proc: 实施进度,仅筛选 div_proc='实施'的记录
|
||||
3. 调用tushare 的trade_cal接口，查询起止日期内的所有交易日
|
||||
4. 建一个新的df,在起止日期内填充，规则：
|
||||
- 下列运算过程中将起始时间往前推 gap_days,赋值360天。运算结束后，截取起止时间内的数据
|
||||
- trade_date: 以交易日历为准，填充所有交易日。
|
||||
- 填入当ex_date=trade_date的日期，填入：ex_date,cash_div_tax，其余日期ex_date留空，cash_div_tax 置 0
|
||||
- cash_div_year:逐行计算填入，以trade_date往前计算过去gap_days天内的cash_div_tax之和
|
||||
5. 调用tushare的个股日线行情接口：https://tushare.pro/document/2?doc_id=27，查询起止日期内的个股日线行情数据，包含以下字段：
|
||||
- ts_code: 股票代码
|
||||
- trade_date: 交易日期
|
||||
- close: 收盘价
|
||||
6. 将分红数据和日线行情数据按交易日期合并，得到最终结果，包含以下字段：
|
||||
- ts_code: 股票代码
|
||||
- trade_date: 交易日期
|
||||
- close: 收盘价
|
||||
- cach_div_tax: 每股现金分红（含税）
|
||||
- cach_div_year: 每股现金TTM年度分红（含税）
|
||||
- div_yield: 股息率，计算公式为 (cach_div_year / close) * 100，保留PRECISION_CONFIG位小数，如果cash_div_year为0则div_yield也为0
|
||||
7. 返回最终结果的DataFrame
|
||||
'''
|
||||
|
||||
|
||||
# 初始化tushare
|
||||
from .data_source import get_tushare_pro
|
||||
pro = get_tushare_pro()
|
||||
|
||||
def get_dividend_yield(ts_code, start_date, end_date):
|
||||
# 检查日期范围是否超过当前日期，若超过则调整为当前日期
|
||||
today = datetime.now().strftime("%Y%m%d")
|
||||
if end_date > today:
|
||||
end_date = today
|
||||
if start_date > today:
|
||||
start_date = today
|
||||
|
||||
gap_days = 360 # 定义TTM计算窗口期为360天
|
||||
|
||||
# 1. 获取分红数据：查询指定股票的分红信息
|
||||
df_div = pro.dividend(ts_code=ts_code, fields='ts_code,end_date,ann_date,ex_date,cash_div_tax,div_proc')
|
||||
# 筛选实施状态的分红记录
|
||||
df_div = df_div[df_div['div_proc'] == '实施']
|
||||
print(f"分红原始数据：\n {df_div.to_string()} ")
|
||||
|
||||
# 2. 获取交易日历：查询指定日期范围内的开盘日
|
||||
df_cal = pro.trade_cal(exchange='', start_date=start_date, end_date=end_date, is_open='1')
|
||||
trade_dates = df_cal['cal_date'].tolist()
|
||||
|
||||
# 3. 创建基础DataFrame：将计算窗口期向前扩展gap_days天
|
||||
extended_start = pd.to_datetime(start_date) - pd.Timedelta(days=gap_days)
|
||||
extended_start_str = extended_start.strftime('%Y%m%d')
|
||||
# 获取扩展后的交易日历
|
||||
df_cal_ext = pro.trade_cal(exchange='', start_date=extended_start_str, end_date=end_date, is_open='1')
|
||||
df_base = pd.DataFrame({'trade_date': df_cal_ext['cal_date']})
|
||||
|
||||
# 4. 合并分红数据：将分红信息按除权除息日合并到基础交易日历
|
||||
df_base = df_base.merge(df_div[['ex_date', 'cash_div_tax']],
|
||||
left_on='trade_date', right_on='ex_date', how='left')
|
||||
# 填充空值：无分红日期现金分红设为0
|
||||
df_base['cash_div_tax'] = df_base['cash_div_tax'].fillna(0)
|
||||
|
||||
# 5. 计算TTM年度分红：滚动计算过去gap_days天的现金分红总和
|
||||
# 修正滚动窗口计算：使用固定窗口大小，确保在窗口期内正确累加
|
||||
df_base['cash_div_year'] = df_base['cash_div_tax'].rolling(window=gap_days, min_periods=0).sum()
|
||||
|
||||
|
||||
print(f"分红填充数据S1：\n {df_base.to_string()} ")
|
||||
# 6. 截取指定日期范围：保留原始查询日期范围内的数据
|
||||
df_base = df_base[df_base['trade_date'] >= start_date]
|
||||
print(f"分红填充数据S2：\n {df_base.to_string()} ")
|
||||
# 7. 获取日线行情数据：查询指定股票的日线收盘价
|
||||
df_daily = pro.daily(ts_code=ts_code, start_date=start_date, end_date=end_date,
|
||||
fields='ts_code,trade_date,close')
|
||||
|
||||
# 8. 合并数据：将行情数据与分红数据按交易日合并
|
||||
result = df_base.merge(df_daily, on='trade_date', how='left')
|
||||
# 填充股票代码：确保所有行都有股票代码
|
||||
result['ts_code'] = result['ts_code'].fillna(ts_code)
|
||||
|
||||
# 9. 计算股息率：当TTM分红>0时计算（分红/收盘价）*100，否则为0
|
||||
result['div_yield'] = np.where(
|
||||
result['cash_div_year'] > 0,
|
||||
(result['cash_div_year'] / result['close']) * 100,
|
||||
0
|
||||
)
|
||||
# 精度处理：保留配置指定的小数位数
|
||||
result['div_yield'] = result['div_yield'].round(PRECISION_CONFIG)
|
||||
|
||||
# 10. 整理列顺序：选择并排列最终输出的列
|
||||
result = result[['ts_code', 'trade_date', 'close', 'cash_div_tax', 'cash_div_year', 'div_yield']]
|
||||
|
||||
return result
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试代码
|
||||
test_code = "600900.SH"
|
||||
test_start = "20230101"
|
||||
test_end = "20251231"
|
||||
|
||||
try:
|
||||
result = get_dividend_yield(test_code, test_start, test_end)
|
||||
print(f"股票 {test_code} 的分红股息率数据:")
|
||||
print(result.head(100))
|
||||
print(f"\n数据形状: {result.shape}")
|
||||
print(f"\n数据列名: {result.columns.tolist()}")
|
||||
|
||||
# 检查是否有分红数据
|
||||
if not result.empty:
|
||||
print(f"\n股息率统计:")
|
||||
print(result['div_yield'].describe())
|
||||
else:
|
||||
print("未找到分红数据")
|
||||
|
||||
except Exception as e:
|
||||
print(f"测试过程中出现错误: {e}")
|
||||
@@ -0,0 +1,223 @@
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from .stock_utils import *
|
||||
from .smoothBrush import smooth_dataframe_brush
|
||||
except (ImportError, SystemError):
|
||||
from config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from stock_utils import *
|
||||
from smoothBrush import smooth_dataframe_brush
|
||||
|
||||
from .data_source import get_tushare_pro
|
||||
|
||||
pro = get_tushare_pro()
|
||||
'''
|
||||
给定个股代码和起止日期：
|
||||
1. 调用tushare接口查询个股分红数据，接口文档：https://tushare.pro/document/2?doc_id=103 查询日期范围内的所有分红记录
|
||||
2. 返回调整后的分红记录，包含以下字段：
|
||||
- ts_code: 股票代码
|
||||
- end_date: 分红年度
|
||||
- ann_date: 公告日期
|
||||
- ex_date: 除权除息日
|
||||
- cash_div_tax: 每股现金分红（含税）
|
||||
- div_proc: 实施进度,仅筛选 div_proc='实施'的记录
|
||||
3. 调用tushare 的trade_cal接口，查询起止日期内的所有交易日
|
||||
4. 建一个新的df,在起止日期内填充，规则：
|
||||
- 下列运算过程中将起始时间往前推 gap_days,赋值360天。运算结束后，截取起止时间内的数据
|
||||
- trade_date: 以交易日历为准，填充所有交易日。
|
||||
- 填入当ex_date=trade_date的日期，填入：ex_date,cash_div_tax，其余日期ex_date留空，cash_div_tax 置 0
|
||||
- cash_div_year:逐行计算填入，以trade_date往前计算过去gap_days天内的cash_div_tax之和
|
||||
5. 调用tushare的个股日线行情接口：https://tushare.pro/document/2?doc_id=27，查询起止日期内的个股日线行情数据，包含以下字段：
|
||||
- ts_code: 股票代码
|
||||
- trade_date: 交易日期
|
||||
- close: 收盘价
|
||||
6. 将分红数据和日线行情数据按交易日期合并，得到最终结果，包含以下字段：
|
||||
- ts_code: 股票代码
|
||||
- trade_date: 交易日期
|
||||
- close: 收盘价
|
||||
- cach_div_tax: 每股现金分红（含税）
|
||||
- cach_div_year: 每股现金TTM年度分红（含税）
|
||||
- div_yield: 股息率，计算公式为 (cach_div_year / close) * 100，保留PRECISION_CONFIG位小数，如果cash_div_year为0则div_yield也为0
|
||||
7. 返回最终结果的DataFrame
|
||||
'''
|
||||
def analyze_stock_dividend_and_price(ts_code, start_date=START_DATE, end_date=END_DATE):
|
||||
# 检查日期范围是否超过当前日期，若超过则调整为当前日期
|
||||
start_date=date_format_correction(start_date)
|
||||
end_date=date_format_correction(end_date)
|
||||
today = datetime.now().strftime("%Y%m%d")
|
||||
if end_date > today: end_date = today
|
||||
if start_date > today: start_date = today
|
||||
|
||||
GAP_DAYS = 360 # 定义TTM计算窗口期为360天
|
||||
n = 15 # 定义向前填充的最大非零值个数
|
||||
"""
|
||||
分析个股分红与行情数据。
|
||||
|
||||
参数:
|
||||
ts_code (str): 股票代码，例如 '000001.SZ'。
|
||||
start_date (str): 起始日期，格式 'YYYYMMDD'。
|
||||
end_date (str): 结束日期，格式 'YYYYMMDD'。
|
||||
|
||||
返回:
|
||||
pd.DataFrame: 包含合并后数据的DataFrame。
|
||||
"""
|
||||
|
||||
# 1. 调用tushare接口查询个股分红数据
|
||||
try:
|
||||
df_div_raw = pro.dividend(ts_code=ts_code)
|
||||
# 2. 筛选并调整分红记录
|
||||
# 筛选实施进度为'实施'的记录，并在指定日期范围内
|
||||
df_div_filtered = df_div_raw[
|
||||
(df_div_raw['div_proc'] == '实施')
|
||||
].copy()
|
||||
|
||||
df_div_adjusted = df_div_filtered[[
|
||||
'ts_code', 'end_date', 'ann_date', 'ex_date', 'cash_div_tax'
|
||||
]].reset_index(drop=True)
|
||||
|
||||
except Exception as e:
|
||||
print(f"获取或处理分红数据时出错: {e}")
|
||||
return pd.DataFrame() # 返回空DataFrame
|
||||
|
||||
# 3. 调用tushare 的trade_cal接口，查询交易日
|
||||
try:
|
||||
# 运算时起始时间往前推 GAP_DAYS
|
||||
calc_start_date = pd.to_datetime(start_date) - pd.Timedelta(days=GAP_DAYS)
|
||||
calc_start_date_str = calc_start_date.strftime('%Y%m%d')
|
||||
|
||||
df_cal = pro.trade_cal(exchange='', start_date=calc_start_date_str, end_date=end_date)
|
||||
# 筛选交易日
|
||||
trade_dates_all = df_cal[df_cal['is_open'] == 1]['cal_date'].sort_values().tolist()
|
||||
|
||||
except Exception as e:
|
||||
print(f"获取交易日历时出错: {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
# 4. 建立新df并填充
|
||||
df_div_processed = pd.DataFrame({'trade_date': trade_dates_all})
|
||||
df_div_processed['trade_date'] = pd.to_datetime(df_div_processed['trade_date'], format='%Y%m%d')
|
||||
|
||||
# 将原始分红数据的ex_date也转为datetime以便合并
|
||||
df_div_adjusted['ex_date'] = pd.to_datetime(df_div_adjusted['ex_date'], format='%Y%m%d')
|
||||
|
||||
# 合并分红数据到交易日历
|
||||
df_merged_temp = df_div_processed.merge(df_div_adjusted[['ex_date', 'cash_div_tax']],
|
||||
left_on='trade_date', right_on='ex_date', how='left')
|
||||
df_merged_temp.drop('ex_date', axis=1, inplace=True)
|
||||
# 填充空值
|
||||
df_merged_temp['cash_div_tax'] = df_merged_temp['cash_div_tax'].fillna(0.0)
|
||||
|
||||
# 计算 cash_div_year (TTM)
|
||||
df_merged_temp = df_merged_temp.sort_values('trade_date').reset_index(drop=True)
|
||||
# 使用滚动窗口计算过去 GAP_DAYS 天的总和
|
||||
# rolling的window参数是基于行数的，所以我们需要先确保日期是连续的交易日
|
||||
# 由于trade_date已经是交易日，我们可以直接使用rolling
|
||||
# 但需要处理时间窗口，确保是360天而不是360行（因为可能有节假日）
|
||||
# 更精确的方法是使用一个自定义函数来累加过去360天内的值
|
||||
|
||||
# 使用更精确的日期差计算
|
||||
def calculate_ttm_div(row_idx):
|
||||
current_date = df_merged_temp.loc[row_idx, 'trade_date']
|
||||
start_window_date = current_date - pd.Timedelta(days=GAP_DAYS)
|
||||
# 筛选出窗口期内的记录
|
||||
mask = (df_merged_temp['trade_date'] > start_window_date) & (df_merged_temp['trade_date'] <= current_date)
|
||||
return df_merged_temp.loc[mask, 'cash_div_tax'].sum()
|
||||
|
||||
df_merged_temp['cash_div_year'] = [calculate_ttm_div(i) for i in range(len(df_merged_temp))]
|
||||
|
||||
|
||||
# 截取原始请求的起止时间内的数据
|
||||
start_date_dt = pd.to_datetime(start_date, format='%Y%m%d')
|
||||
end_date_dt = pd.to_datetime(end_date, format='%Y%m%d')
|
||||
df_div_final = df_merged_temp[
|
||||
(df_merged_temp['trade_date'] >= start_date_dt) &
|
||||
(df_merged_temp['trade_date'] <= end_date_dt)
|
||||
].copy()
|
||||
df_div_final['trade_date'] = df_div_final['trade_date'].dt.strftime('%Y%m%d')
|
||||
|
||||
# 5. 调用tushare的个股日线行情接口
|
||||
try:
|
||||
df_daily = pro.daily(ts_code=ts_code, start_date=start_date, end_date=end_date)
|
||||
df_daily = df_daily[['ts_code', 'trade_date', 'close']].sort_values('trade_date').reset_index(drop=True)
|
||||
except Exception as e:
|
||||
print(f"获取日线行情数据时出错: {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
# 6. 合并分红数据和日线行情数据
|
||||
df_result = df_daily.merge(df_div_final[['trade_date', 'cash_div_tax', 'cash_div_year']],
|
||||
on='trade_date', how='left')
|
||||
|
||||
# 填充因合并可能产生的NaN（例如，某日有行情但无分红记录）
|
||||
df_result['cash_div_tax'] = df_result['cash_div_tax'].fillna(0.0)
|
||||
df_result['cash_div_year'] = df_result['cash_div_year'].fillna(0.0)
|
||||
|
||||
|
||||
'''
|
||||
向前填充cash_div_year（最多填充最近n个非零值）
|
||||
如果遇到0值，依次往下查询直到查询到非0数字为止，如果数字个数<=n个，就置last_valid_value, 否则保持不变
|
||||
'''
|
||||
# 确保按trade_date倒序遍历（日期从大到小）
|
||||
df_result = df_result.sort_values('trade_date', ascending=False).reset_index(drop=True)
|
||||
|
||||
last_valid_value = None # 初始化最后一个有效值变量
|
||||
for idx in range(len(df_result)): # 遍历DataFrame的每一行
|
||||
current_value = df_result.loc[idx, 'cash_div_year'] # 获取当前行的TTM分红值
|
||||
if current_value > 0: # 如果当前值大于0
|
||||
last_valid_value = current_value # 更新最后一个有效值
|
||||
else:
|
||||
# 向下查找最多n个位置内的非零值
|
||||
found_value = None # 初始化找到的值
|
||||
search_count = 0 # 初始化搜索计数
|
||||
# 从下一行开始搜索，最多搜索n行
|
||||
for search_idx in range(idx + 1, min(idx + n + 1, len(df_result))):
|
||||
search_value = df_result.loc[search_idx, 'cash_div_year'] # 获取搜索行的值
|
||||
search_count += 1 # 增加搜索计数
|
||||
if search_value > 0: # 如果找到非零值
|
||||
found_value = search_value # 记录找到的值
|
||||
break # 跳出搜索循环
|
||||
|
||||
# 如果找到非零值且在n个位置内
|
||||
if found_value is not None and search_count <= n:
|
||||
df_result.loc[idx, 'cash_div_year'] = found_value
|
||||
last_valid_value = found_value
|
||||
elif last_valid_value is not None:
|
||||
df_result.loc[idx, 'cash_div_year'] = last_valid_value
|
||||
|
||||
# 毛刺平滑处理 cash_div_year 列
|
||||
df_result=smooth_dataframe_brush(df_result, target_columns=['cash_div_year'], window_size=31, threshold_factor=0.5, max_brush_length=15 )
|
||||
# 恢复原始日期顺序
|
||||
df_result = df_result.sort_values('trade_date').reset_index(drop=True)
|
||||
|
||||
# 计算股息率
|
||||
df_result['div_yield'] = 0.0
|
||||
mask_non_zero_price = df_result['close'] > 0
|
||||
mask_non_zero_div_year = df_result['cash_div_year'] > 0
|
||||
# 只对收盘价大于0且TTM分红大于0的记录计算股息率
|
||||
valid_mask = mask_non_zero_price & mask_non_zero_div_year
|
||||
df_result.loc[valid_mask, 'div_yield'] = (
|
||||
(df_result.loc[valid_mask, 'cash_div_year'] / df_result.loc[valid_mask, 'close']) * 100
|
||||
).round(PRECISION_CONFIG)
|
||||
|
||||
# 7. 返回最终结果
|
||||
# 重命名字段以匹配要求 (注意: 题目中'cach_div_tax'应为'cash_div_tax')
|
||||
#df_result.rename(columns={'cash_div_tax': 'cach_div_tax', 'cash_div_year': 'cach_div_year'}, inplace=True)
|
||||
|
||||
final_columns = ['ts_code', 'trade_date', 'close', 'cash_div_tax', 'cash_div_year', 'div_yield']
|
||||
df_final = df_result[final_columns]
|
||||
|
||||
return df_final
|
||||
|
||||
|
||||
|
||||
# --- 示例用法 ---
|
||||
if __name__ == "__main__":
|
||||
start_dt = '2020-01-01'
|
||||
end_dt = '2025-12-31'
|
||||
ts_code = '000001.SZ'
|
||||
result = analyze_stock_dividend_and_price(ts_code, start_dt, end_dt)
|
||||
print(f"股票 {ts_code} 的分红与行情数据分析结果:")
|
||||
print(result.head(10))
|
||||
print(f"\n数据总行数: {len(result)}")
|
||||
@@ -0,0 +1,393 @@
|
||||
import pandas as pd
|
||||
# 将项目根目录添加到 sys.path
|
||||
#project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
#sys.path.append(project_root)
|
||||
|
||||
# 导入当站目录的config文件
|
||||
try:
|
||||
# 尝试相对导入（作为包的一部分）
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from .stock_utils import *
|
||||
except (ImportError, SystemError):
|
||||
# 失败则使用绝对导入（直接运行脚本）
|
||||
from config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from stock_utils import *
|
||||
|
||||
from .data_source import get_tushare_pro
|
||||
pro = get_tushare_pro()
|
||||
def getStockEp(TS_CODE,start_date=START_DATE,end_date=END_DATE):
|
||||
"""
|
||||
从tushare接口获取单只股票的财务数据，计算并填充每日的每股收益(EP)指标。
|
||||
Parameters:
|
||||
TS_CODE (str): 股票代码,格式为 '股票代码.SZ' 或 '股票代码.SH',例如 '000001.SZ'
|
||||
START_DATE (str): 开始日期,格式为 'YYYYMMDD'
|
||||
END_DATE (str): 结束日期,格式为 'YYYYMMDD'
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含以下字段的DataFrame:
|
||||
ts_code: 股票代码
|
||||
ann_date: 财报公告日期
|
||||
end_date: 财报结束日期/填充日期
|
||||
basic_eps: 基本每股收益
|
||||
diluted_eps: 稀释每股收益
|
||||
trade_date: 实际交易日(来自日线数据)
|
||||
close: 收盘价
|
||||
basic_ep: 基本EP值(basic_eps/close)
|
||||
diluted_ep: 稀释EP值(diluted_eps/close)
|
||||
|
||||
Raises:
|
||||
Exception: 如果从Tushare接口获取数据时发生错误。
|
||||
"""
|
||||
|
||||
'''
|
||||
1. 调用get_trading_dates函数，填充完整df_income 变量内部end_date，规则为：以end_date为限，往前天从到上一个财报日期为止(不包含上一个财报日期)，期间填充basic_eps,diluted_eps值为报告日的相同值
|
||||
2. 新增一个basic_ep列，数据为:basic_eps/当天交易日期的股价，当天交易日期的股价来自df_daily的close列
|
||||
3. 新增一个diluted_ep列，数据为:diluted_eps/当天交易日期的股价，当天交易日期的股价来自df_daily的close列
|
||||
'''
|
||||
try:
|
||||
# 获取股票财务数据
|
||||
df_income = pro.income(ts_code=TS_CODE, start_date=start_date, end_date=end_date,
|
||||
fields='ts_code,ann_date,end_date,basic_eps,diluted_eps')
|
||||
|
||||
# 填充df_income的end_date区间数据
|
||||
if not df_income.empty:
|
||||
# 按end_date分组并排序财报数据
|
||||
df_income = df_income.sort_values('end_date')
|
||||
grouped = df_income.groupby('end_date')
|
||||
# 存储填充后的数据
|
||||
filled_data = []
|
||||
|
||||
# 遍历每个财报期间
|
||||
for end_date, group in grouped:
|
||||
# 获取该日期到上一个财报日的所有交易日期
|
||||
trading_dates = get_trading_dates(end_date)
|
||||
# 填充数据
|
||||
for date in trading_dates:
|
||||
# 将日期对象转换为字符串格式（YYYYMMDD）
|
||||
# 如果date是日期对象（有strftime方法），则调用strftime格式化
|
||||
# 否则直接使用原值（假设已经是字符串格式）
|
||||
date_str = date.strftime('%Y%m%d') if hasattr(date, 'strftime') else date
|
||||
filled_row = {
|
||||
'ts_code': TS_CODE,
|
||||
'ann_date': group['ann_date'].iloc[0],
|
||||
'end_date': date_str,
|
||||
'basic_eps': group['basic_eps'].iloc[0],
|
||||
'diluted_eps': group['diluted_eps'].iloc[0]
|
||||
}
|
||||
filled_data.append(filled_row)
|
||||
|
||||
# 合并填充后的数据
|
||||
df_income = pd.DataFrame(filled_data)
|
||||
# 获取股票日线数据以获取股价
|
||||
min_end_date = df_income['end_date'].min()
|
||||
df_daily = pro.daily(ts_code=TS_CODE, start_date=min_end_date, end_date=END_DATE)
|
||||
|
||||
# 2. 计算basic_ep和diluted_ep
|
||||
if not df_income.empty and not df_daily.empty:
|
||||
# 修改原因：原代码使用left_on='end_date'和right_on='trade_date'导致匹配失败
|
||||
# 解决方案：将df_daily的trade_date转换为字符串格式再合并
|
||||
df_daily['trade_date_str'] = df_daily['trade_date'].astype(str)
|
||||
df_income = pd.merge(
|
||||
df_income,
|
||||
df_daily[['trade_date_str', 'close']],
|
||||
left_on='end_date',
|
||||
right_on='trade_date_str',
|
||||
how='left'
|
||||
)
|
||||
# 将合并后的trade_date_str重命名为trade_date
|
||||
df_income.rename(columns={'trade_date_str': 'trade_date'}, inplace=True)
|
||||
|
||||
# 计算basic_ep和diluted_ep
|
||||
df_income['basic_ep'] = (df_income['basic_eps'] / df_income['close']).round(PRECISION_CONFIG)
|
||||
df_income['diluted_ep'] = (df_income['diluted_eps'] / df_income['close']).round(PRECISION_CONFIG)
|
||||
#print("df_income:",df_income)
|
||||
# 按日期截断
|
||||
#df_income = df_income[(df_income['trade_date'] >= START_DATE) & (df_income['trade_date'] <= END_DATE)]
|
||||
|
||||
# 数据修正
|
||||
cols = ['basic_eps', 'diluted_eps', 'close', 'basic_ep', 'diluted_ep']
|
||||
df_income = dataCorrect(df_income, cols).sort_values('trade_date')
|
||||
|
||||
return df_income
|
||||
|
||||
except Exception as e:
|
||||
# 捕获并处理异常
|
||||
print(f"错误: 获取股票 {TS_CODE} 的EP数据时发生错误: {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
def getStockEp_ttm(TS_CODE,start_date=START_DATE,end_date=END_DATE):
|
||||
"""
|
||||
获取股票的TTM(最近12个月)每股收益与股价比率(EP)数据
|
||||
|
||||
Parameters:
|
||||
TS_CODE (str): 股票代码,格式为 '股票代码.SZ' 或 '股票代码.SH',例如 '000001.SZ'
|
||||
START_DATE (str): 开始日期,格式为 'YYYYMMDD'
|
||||
END_DATE (str): 结束日期,格式为 'YYYYMMDD'
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含以下字段的DataFrame:
|
||||
ts_code: 股票代码
|
||||
trade_date: 交易日
|
||||
close: 收盘价
|
||||
basic_ep_ttm: 基本EP_TTM值(basic_eps_q_ttm/close)
|
||||
diluted_ep_ttm: 稀释EP_TTM值(diluted_eps_q_ttm/close)
|
||||
basic_eps_q_ttm: 基本每股收益TTM值
|
||||
diluted_eps_q_ttm: 稀释每股收益TTM值
|
||||
|
||||
Raises:
|
||||
Exception: 如果从Tushare接口获取数据时发生错误
|
||||
"""
|
||||
try:
|
||||
start_date=date_format_correction(start_date) #confirm date as yyyymmdd
|
||||
end_date=date_format_correction(end_date) #confirm date as yyyymmdd
|
||||
df_eps = get_quarterly_eps(tscodeCheck(TS_CODE), start_date, end_date)
|
||||
df_eps_ttm = calculate_ttm_eps(df_eps)
|
||||
df_eps_ttm=fill_trading_dates_with_eps(df_eps_ttm,start_date=start_date,end_date=end_date)
|
||||
df_daily = pro.daily(ts_code=TS_CODE, start_date=start_date, end_date=end_date)
|
||||
df_eps_ttm = pd.merge(
|
||||
df_eps_ttm,
|
||||
df_daily[['trade_date', 'close']],
|
||||
on='trade_date',
|
||||
how='left'
|
||||
)
|
||||
df_eps_ttm['basic_ep_ttm'] = (100*df_eps_ttm['basic_eps_q_ttm'] / df_eps_ttm['close']).round(PRECISION_CONFIG)
|
||||
df_eps_ttm['diluted_ep_ttm'] = (100*df_eps_ttm['diluted_eps_q_ttm'] / df_eps_ttm['close']).round(PRECISION_CONFIG)
|
||||
# 比较最大日期并补充数据, 解决当ep不存在，close值也不会显示的问题
|
||||
if not df_eps_ttm.empty and not df_daily.empty:
|
||||
max_eps_date = df_eps_ttm['trade_date'].max()
|
||||
max_daily_date = df_daily['trade_date'].max()
|
||||
print(f"max_eps_date:{max_eps_date}")
|
||||
print(f"max_daily_date:{max_daily_date}")
|
||||
if max_eps_date != max_daily_date:
|
||||
# 获取需要补充的日期范围
|
||||
mask = (df_daily['trade_date'] > max_eps_date) & (df_daily['trade_date'] <= max_daily_date)
|
||||
additional_data = df_daily.loc[mask, ['ts_code', 'trade_date', 'close']].copy()
|
||||
|
||||
# 补充空列
|
||||
for col in df_eps_ttm.columns:
|
||||
if col not in ['ts_code', 'trade_date', 'close']:
|
||||
additional_data[col] = ''
|
||||
print(additional_data)
|
||||
# 合并数据
|
||||
df_eps_ttm = pd.concat([df_eps_ttm, additional_data], ignore_index=True)
|
||||
|
||||
cols=["close","basic_ep_ttm","diluted_ep_ttm","basic_eps_q_ttm","diluted_eps_q_ttm"]
|
||||
df_eps_ttm = dataCorrect(df_eps_ttm,cols)
|
||||
return df_eps_ttm[['ts_code', 'trade_date', 'close', 'basic_ep_ttm', 'diluted_ep_ttm', 'basic_eps_q_ttm', 'diluted_eps_q_ttm']]
|
||||
except Exception as e:
|
||||
# 捕获并处理异常
|
||||
print(f"错误: 获取股票 {TS_CODE} 的EP_TTM数据时发生错误: {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
def get_quarterly_eps(TS_CODE, start_date=START_DATE, end_date=END_DATE):
|
||||
"""
|
||||
获取单季度每股收益数据
|
||||
Parameters:
|
||||
TS_CODE (str): 股票代码
|
||||
START_DATE (str): 开始日期(YYYYMMDD)
|
||||
END_DATE (str): 结束日期(YYYYMMDD)
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含以下字段的DataFrame:
|
||||
ts_code: 股票代码
|
||||
ann_date: 财报公告日期
|
||||
end_date: 财报结束日期(YYYYMMDD格式)
|
||||
basic_eps: 累计基本每股收益
|
||||
diluted_eps: 累计稀释每股收益
|
||||
basic_eps_q: 单季度基本每股收益
|
||||
diluted_eps_q: 单季度稀释每股收益
|
||||
"""
|
||||
|
||||
try:
|
||||
# 扩展日期范围往前三个季度
|
||||
extended_start = (pd.to_datetime(start_date) - pd.DateOffset(months=12)).strftime('%Y%m%d')
|
||||
|
||||
# 获取原始财务数据
|
||||
df = pro.income(ts_code=TS_CODE, start_date=extended_start, end_date=end_date,
|
||||
fields='ts_code,ann_date,end_date,basic_eps,diluted_eps')
|
||||
|
||||
if df.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
# 按财报日期排序并去重（解决重复数据问题）
|
||||
df = df.sort_values('end_date').drop_duplicates(subset=['end_date'], keep='last') # 修改原因：确保每个end_date只保留最新数据
|
||||
# 计算单季度数据
|
||||
df['basic_eps_q'] = df['basic_eps']
|
||||
df['diluted_eps_q'] = df['diluted_eps']
|
||||
|
||||
# 非第一季度数据需要减去上季度数据
|
||||
mask = ~df['end_date'].str.endswith('0331')
|
||||
df.loc[mask, 'basic_eps_q'] = df['basic_eps'].diff()
|
||||
df.loc[mask, 'diluted_eps_q'] = df['diluted_eps'].diff()
|
||||
|
||||
# 过滤掉非季报数据（保留3/6/9/12月数据）
|
||||
df = df[df['end_date'].str.endswith(('0331', '0630', '0930', '1231'))]
|
||||
|
||||
# 删除最小日期的数据行
|
||||
if not df.empty:
|
||||
df = df[df['end_date'] != df['end_date'].min()]
|
||||
|
||||
df['basic_eps_q'] = df['basic_eps_q'].round(PRECISION_CONFIG)
|
||||
df['diluted_eps_q'] = df['diluted_eps_q'].round(PRECISION_CONFIG)
|
||||
# 重置行号并保留原始EPS值
|
||||
#return df[['ts_code', 'ann_date', 'end_date', 'basic_eps', 'diluted_eps', 'basic_eps_q', 'diluted_eps_q']].reset_index(drop=True)
|
||||
#重置行号，去掉原始EPS值
|
||||
return df[['ts_code', 'ann_date', 'end_date', 'basic_eps_q', 'diluted_eps_q']].reset_index(drop=True)
|
||||
|
||||
except Exception as e:
|
||||
print(f"获取季度EPS数据出错: {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
def calculate_ttm_eps(df):
|
||||
"""
|
||||
计算EPS指标的TTM(最近12个月)值
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): get_quarterly_eps函数返回的DataFrame，包含以下列:
|
||||
- ts_code: 股票代码
|
||||
- ann_date: 公告日期
|
||||
- end_date: 财报结束日期
|
||||
- basic_eps: 基本每股收益(累计)
|
||||
- diluted_eps: 稀释每股收益(累计)
|
||||
- basic_eps_q: 单季度基本每股收益
|
||||
- diluted_eps_q: 单季度稀释每股收益
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含原始数据和TTM计算结果的DataFrame，新增以下列:
|
||||
- basic_eps_ttm: 基本每股收益TTM值
|
||||
- diluted_eps_ttm: 稀释每股收益TTM值
|
||||
- basic_eps_q_ttm: 单季度基本每股收益TTM值
|
||||
- diluted_eps_q_ttm: 单季度稀释每股收益TTM值
|
||||
"""
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
try:
|
||||
# 确保数据按end_date降序排列
|
||||
df = df.sort_values('end_date', ascending=False).reset_index(drop=True)
|
||||
|
||||
# 初始化TTM结果列
|
||||
#df['basic_eps_ttm'] = None
|
||||
#df['diluted_eps_ttm'] = None
|
||||
df['basic_eps_q_ttm'] = None
|
||||
df['diluted_eps_q_ttm'] = None
|
||||
|
||||
# 遍历每一行数据计算TTM
|
||||
for i in range(len(df)):
|
||||
# 检查是否有足够的后续数据(至少3个季度)
|
||||
if i + 3 >= len(df):
|
||||
continue # 数据不足，跳过计算
|
||||
|
||||
# 计算TTM值(当前季度+后续3个季度)
|
||||
#df.at[i, 'basic_eps_ttm'] = df.loc[i:i+3, 'basic_eps'].sum()
|
||||
# df.at[i, 'diluted_eps_ttm'] = df.loc[i:i+3, 'diluted_eps'].sum()
|
||||
df.at[i, 'basic_eps_q_ttm'] = df.loc[i:i+3, 'basic_eps_q'].sum()
|
||||
df.at[i, 'diluted_eps_q_ttm'] = df.loc[i:i+3, 'diluted_eps_q'].sum()
|
||||
|
||||
# 四舍五入保留指定小数位数
|
||||
ttm_cols = [ 'basic_eps_q_ttm', 'diluted_eps_q_ttm']
|
||||
df[ttm_cols] = df[ttm_cols].round(PRECISION_CONFIG)
|
||||
# 删除最后三行数据，因为最后三行ttm数据为None
|
||||
df = df.iloc[:-3]
|
||||
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
print(f"计算TTM值时出错: {e}")
|
||||
return pd.DataFrame()
|
||||
def fill_trading_dates_with_eps(df_ttm,start_date=START_DATE,end_date=END_DATE):
|
||||
"""
|
||||
填充交易日期并保留EPS值
|
||||
|
||||
参数:
|
||||
df_ttm (pd.DataFrame): calculate_ttm_eps函数返回的DataFrame，包含以下列:
|
||||
- end_date: 财报结束日期(YYYYMMDD格式)
|
||||
- basic_eps_ttm: 基本每股收益TTM值
|
||||
- diluted_eps_ttm: 稀释每股收益TTM值
|
||||
- basic_eps_q_ttm: 单季度基本每股收益TTM值
|
||||
- diluted_eps_q_ttm: 单季度稀释每股收益TTM值
|
||||
|
||||
返回:
|
||||
pd.DataFrame: 包含填充后的交易日期和对应EPS值的DataFrame
|
||||
|
||||
异常处理:
|
||||
- 输入为空DataFrame时直接返回
|
||||
- Tushare接口调用失败时返回原始数据
|
||||
"""
|
||||
if df_ttm.empty:
|
||||
return df_ttm
|
||||
|
||||
try:
|
||||
# 1. 准备数据: 按end_date排序并转换为datetime格式
|
||||
df_ttm = df_ttm.sort_values('end_date')
|
||||
df_ttm['end_date_dt'] = pd.to_datetime(df_ttm['end_date'])
|
||||
# 2. 获取所有需要填充的日期区间
|
||||
date_ranges = []
|
||||
for i in range(len(df_ttm)-1):
|
||||
s_date = df_ttm['end_date_dt'].iloc[i]
|
||||
e_date = df_ttm['end_date_dt'].iloc[i+1]
|
||||
date_ranges.append((s_date, e_date))
|
||||
# 检查是否需要添加最后一个区间
|
||||
if df_ttm['end_date_dt'].max() < pd.to_datetime(end_date):
|
||||
next_report_date = pd.to_datetime(get_next_report_date(df_ttm['end_date_dt'].max()))
|
||||
next_report_date = min(next_report_date, pd.to_datetime(end_date))
|
||||
|
||||
date_ranges.append((df_ttm['end_date_dt'].max(), next_report_date))
|
||||
# 3. 获取交易所交易日历
|
||||
exchange = 'SZSE' if df_ttm['ts_code'].iloc[0].endswith('SZ') else 'SSE' # 可以不用考虑SH,SZ,BJ, 理论上日历应该是一样的
|
||||
trade_cal = pro.trade_cal(exchange=exchange,
|
||||
start_date=start_date,
|
||||
end_date=end_date)
|
||||
|
||||
# 过滤出交易日
|
||||
trade_cal = trade_cal[trade_cal['is_open'] == 1]
|
||||
trade_cal['cal_date_dt'] = pd.to_datetime(trade_cal['cal_date'])
|
||||
|
||||
# 4. 填充每个区间内的交易日
|
||||
filled_data = []
|
||||
#eps_cols = ['basic_eps_ttm', 'diluted_eps_ttm', 'basic_eps_q_ttm', 'diluted_eps_q_ttm']
|
||||
|
||||
# 首先添加原始数据
|
||||
for _, row in df_ttm.iterrows():
|
||||
filled_data.append(row.to_dict())
|
||||
|
||||
# 然后填充区间数据
|
||||
for start_date, end_date in date_ranges:
|
||||
# 获取该区间内的所有交易日
|
||||
mask = (trade_cal['cal_date_dt'] > start_date) & (trade_cal['cal_date_dt'] < end_date)
|
||||
dates_in_range = trade_cal[mask]['cal_date_dt']
|
||||
|
||||
# 使用较小的日期(即start_date)的EPS值填充
|
||||
ref_row = df_ttm[df_ttm['end_date_dt'] == start_date].iloc[0]
|
||||
|
||||
for date in dates_in_range:
|
||||
new_row = ref_row.copy()
|
||||
new_row['end_date'] = date.strftime('%Y%m%d')
|
||||
new_row['end_date_dt'] = date
|
||||
filled_data.append(new_row)
|
||||
|
||||
# 5. 转换为DataFrame并整理
|
||||
result = pd.DataFrame(filled_data)
|
||||
result = result.sort_values('end_date_dt')
|
||||
|
||||
# 删除临时列并重置索引
|
||||
result = result.drop(columns=['end_date_dt']).reset_index(drop=True)
|
||||
result = result.drop(columns=['ann_date']) #去掉ann_date列
|
||||
result = result.rename(columns={'end_date': 'trade_date'}) #重命名 end_date 为trade_date
|
||||
# 过滤掉非交易日
|
||||
result = result[result['trade_date'].isin(trade_cal['cal_date'])]
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"填充交易日期时出错: {e}, 返回原始数据")
|
||||
return df_ttm
|
||||
if __name__ == '__main__':
|
||||
#df = getStockEp(tscodeCheck('688469'))
|
||||
'''df = get_quarterly_eps(tscodeCheck('688556'), '20230101', END_DATE)
|
||||
print(df)
|
||||
df2 = calculate_ttm_eps(df)
|
||||
print(df2)
|
||||
df3=fill_trading_dates_with_eps(df2)
|
||||
print(df3)'''
|
||||
df4= getStockEp_ttm(tscodeCheck('300316'), '20230101', END_DATE)
|
||||
print(df4)
|
||||
@@ -0,0 +1,288 @@
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from .stock_utils import *
|
||||
except (ImportError, SystemError):
|
||||
from config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
from stock_utils import *
|
||||
|
||||
from .data_source import get_tushare_pro
|
||||
|
||||
|
||||
class FinanceData:
|
||||
def __init__(self, token=None):
|
||||
"""
|
||||
初始化 Tushare 接口
|
||||
:param token: 可选，不再使用（保留兼容）
|
||||
"""
|
||||
self.pro = get_tushare_pro()
|
||||
self.ts_code = None
|
||||
self.fin_date = None
|
||||
self.unit_factor = 100000000 # 单位换算因子 (元 -> 亿)
|
||||
|
||||
def get_finance_data(self):
|
||||
"""
|
||||
获取完整的财务数据并计算指标
|
||||
:param ts_code: 股票代码 (如 '002273.SZ')
|
||||
:param fin_date: 财报日期 (如 '20220331')
|
||||
:return: 包含财务指标的字典
|
||||
"""
|
||||
|
||||
# 获取各类财务数据 (返回 DataFrame)
|
||||
balance = self.get_balance()
|
||||
pre_balance = self.get_pre_balance()
|
||||
income = self.get_income()
|
||||
cash = self.get_cashflow()
|
||||
balance = balance.where(balance.notna(), 0) # NaN替换为0
|
||||
pre_balance = pre_balance.where(pre_balance.notna(), 0) # NaN替换为0
|
||||
income = income.where(income.notna(), 0) # NaN替换为0
|
||||
cash = cash.where(cash.notna(), 0) # NaN替换为0
|
||||
|
||||
# 初始化结果字典
|
||||
data = {
|
||||
'ts_code': self.ts_code,
|
||||
'period': self.fin_date,
|
||||
'--运营数据--': ''
|
||||
}
|
||||
|
||||
# 运营数据计算
|
||||
cash_equ = cash['c_cash_equ_end_period'].iloc[0]
|
||||
inventories = balance['inventories'].iloc[0]
|
||||
total_assets = balance['total_assets'].iloc[0]
|
||||
|
||||
# 应收账票 = 应收账款 + 应收票据
|
||||
accounts_receiv = float(balance['accounts_receiv'].iloc[0]) + float(balance['notes_receiv'].iloc[0])
|
||||
prepayment = float(balance['prepayment'].iloc[0])
|
||||
|
||||
data.update({
|
||||
'现金额-亿': cash_equ / self.unit_factor,
|
||||
'现金占比率': cash_equ / total_assets,
|
||||
'存货-亿': inventories / self.unit_factor,
|
||||
'存货占比率': inventories / total_assets,
|
||||
'应收账票-亿': accounts_receiv / self.unit_factor,
|
||||
'应收账票占比率': accounts_receiv / total_assets,
|
||||
'预付款-亿': prepayment / self.unit_factor,
|
||||
'预付款占比率': prepayment / total_assets,
|
||||
'运营占比率': (cash_equ + inventories + accounts_receiv + prepayment) / total_assets
|
||||
})
|
||||
|
||||
# 资产分布计算
|
||||
fix_assets = float(balance['fix_assets'].iloc[0]) if 'fix_assets' in balance else 0
|
||||
intan_assets = float(balance['intan_assets'].iloc[0]) if 'intan_assets' in balance else 0
|
||||
lt_eqt_invest = float(balance['lt_eqt_invest'].iloc[0]) if 'lt_eqt_invest' in balance else 0
|
||||
|
||||
data.update({
|
||||
'--资产分布--': '',
|
||||
'固定资产-亿': fix_assets / self.unit_factor,
|
||||
'固定资产占比率': fix_assets / total_assets,
|
||||
'无形资产-亿': intan_assets / self.unit_factor,
|
||||
'无形资产占率': intan_assets / total_assets,
|
||||
'股权投资-亿': lt_eqt_invest / self.unit_factor,
|
||||
'股权投资占比率': lt_eqt_invest / total_assets,
|
||||
'投资占比率': (fix_assets + intan_assets + lt_eqt_invest) / total_assets
|
||||
})
|
||||
|
||||
# 负债分布计算
|
||||
acct_payable = float(balance['acct_payable'].iloc[0]) if 'acct_payable' in balance else 0
|
||||
notes_payable = float(balance['notes_payable'].iloc[0]) if 'notes_payable' in balance else 0
|
||||
adv_receipts = float(balance['adv_receipts'].iloc[0]) if 'adv_receipts' in balance else 0
|
||||
st_borr = float(balance['st_borr'].iloc[0]) if 'st_borr' in balance else 0
|
||||
lt_borr = float(balance['lt_borr'].iloc[0]) if 'lt_borr' in balance else 0
|
||||
bond_payable = float(balance['bond_payable'].iloc[0]) if 'bond_payable' in balance else 0
|
||||
|
||||
biz_liab = acct_payable + notes_payable + adv_receipts
|
||||
fin_liab = st_borr + lt_borr + bond_payable
|
||||
zcfz = (float(balance['total_cur_liab'].iloc[0]) + float(balance['total_ncl'].iloc[0])) / total_assets
|
||||
|
||||
data.update({
|
||||
'--负债分布--': '',
|
||||
'经营负债-亿': biz_liab / self.unit_factor,
|
||||
'经营负债占比率': biz_liab / total_assets,
|
||||
'金融负债-亿': fin_liab / self.unit_factor,
|
||||
'金融负债占比率': fin_liab / total_assets,
|
||||
'资产负债率': zcfz
|
||||
})
|
||||
|
||||
# 运营能力计算
|
||||
total_days = self.get_total_days()
|
||||
oper_cost = income['oper_cost'].iloc[0]
|
||||
revenue = income['revenue'].iloc[0]
|
||||
|
||||
# 存货周转天数 (防除零)
|
||||
#avg_inventories = (float(pre_balance.get('inventories', 0)) + inventories) / 2
|
||||
avg_inventories = (float(pre_balance['inventories'].iloc[0]) + inventories) / 2
|
||||
days_1 = total_days / (oper_cost / avg_inventories) if avg_inventories > 0 else 0
|
||||
# 应收周转天数 (防除零)
|
||||
avg_receiv = (float(pre_balance['accounts_receiv'].iloc[0]) + accounts_receiv) / 2
|
||||
days_2 = total_days / (revenue / avg_receiv) if avg_receiv > 0 else 0
|
||||
|
||||
data.update({
|
||||
'--运营能力--': '',
|
||||
'存货周转天数': days_1,
|
||||
'应收周转天数': days_2,
|
||||
'营业周期': days_1 + days_2
|
||||
})
|
||||
|
||||
# 管理费分布计算
|
||||
gross_profit = revenue - oper_cost
|
||||
gross_margin = gross_profit / revenue if revenue > 0 else 0
|
||||
|
||||
data.update({
|
||||
'--管理费分布--': '',
|
||||
'毛利额': gross_profit / self.unit_factor,
|
||||
'毛利率': gross_margin,
|
||||
'营业税金率': float(income['biz_tax_surchg'].iloc[0]) / float(revenue) if revenue > 0 else 0,
|
||||
'销售费用率': float(income['sell_exp'].iloc[0]) / float(revenue) if revenue > 0 else 0,
|
||||
'研发费用率': float(income['rd_exp'].iloc[0]) / float(revenue) if revenue > 0 else 0,
|
||||
'管理费用率': float(income['admin_exp'].iloc[0]) / float(revenue) if revenue > 0 else 0,
|
||||
'净利润': float(income['n_income'].iloc[0]) / self.unit_factor,
|
||||
'净利润率': float(income['n_income'].iloc[0]) / float(revenue) if revenue > 0 else 0
|
||||
})
|
||||
|
||||
# 权益及回报率计算
|
||||
data.update({
|
||||
'--权益及回报率--': '',
|
||||
'总资产-亿': total_assets / self.unit_factor,
|
||||
'销售收入-亿': revenue / self.unit_factor,
|
||||
'总资产周转率': revenue / total_assets if total_assets > 0 else 0,
|
||||
'总资产回报率': float(income['n_income'].iloc[0]) / total_assets if total_assets > 0 else 0,
|
||||
'权益乘数': 1 / (1 - zcfz) if zcfz < 1 else 0,
|
||||
})
|
||||
|
||||
# 计算ROE (净资产回报率)
|
||||
roa = data['总资产回报率']
|
||||
data['净资产回报率'] = roa * data['权益乘数']
|
||||
|
||||
# 格式化比率数据
|
||||
for key in list(data.keys()):
|
||||
if isinstance(data[key], float):
|
||||
if key.endswith('率'):
|
||||
data[key] = f"{data[key] * 100:.2f}%"
|
||||
else:
|
||||
data[key] = round(data[key], 2)
|
||||
|
||||
return pd.Series(data)
|
||||
|
||||
def get_balance(self):
|
||||
"""获取资产负债表数据"""
|
||||
fields = [
|
||||
'ts_code', 'end_date', 'total_assets', 'fix_assets', 'intan_assets',
|
||||
'lt_eqt_invest', 'inventories', 'accounts_receiv', 'notes_receiv',
|
||||
'prepayment', 'acct_payable', 'notes_payable', 'adv_receipts',
|
||||
'st_borr', 'lt_borr', 'bond_payable', 'total_cur_liab', 'total_ncl'
|
||||
]
|
||||
return self.pro.balancesheet(
|
||||
ts_code=self.ts_code,
|
||||
period=self.fin_date,
|
||||
fields=fields
|
||||
)
|
||||
|
||||
def get_pre_balance(self):
|
||||
"""获取上年度资产负债表数据"""
|
||||
pre_date = f"{int(self.fin_date[:4]) - 1}1231" # 上年末日期
|
||||
fields = ['inventories', 'accounts_receiv', 'notes_receiv']
|
||||
return self.pro.balancesheet(
|
||||
ts_code=self.ts_code,
|
||||
period=pre_date,
|
||||
fields=fields
|
||||
) # 直接返回Series
|
||||
|
||||
def get_income(self):
|
||||
"""获取利润表数据"""
|
||||
fields = [
|
||||
'revenue', 'oper_cost', 'biz_tax_surchg', 'sell_exp', 'fin_exp',
|
||||
'admin_exp', 'n_income', 'rd_exp'
|
||||
]
|
||||
return self.pro.income(
|
||||
ts_code=self.ts_code,
|
||||
period=self.fin_date,
|
||||
fields=fields
|
||||
)
|
||||
|
||||
def get_cashflow(self):
|
||||
"""获取现金流量表数据"""
|
||||
#c_cash_equ_end_period 期末现金及现金等价物余额
|
||||
#end_bal_cash 现金的期末余额
|
||||
fields = ['c_cash_equ_end_period', 'end_bal_cash']
|
||||
return self.pro.cashflow(
|
||||
ts_code=self.ts_code,
|
||||
period=self.fin_date,
|
||||
fields=fields
|
||||
)
|
||||
|
||||
def get_total_days(self):
|
||||
"""根据财报类型返回计算周转率的天数"""
|
||||
quarter = self.fin_date[4:6]
|
||||
return {
|
||||
'03': 90, # Q1
|
||||
'06': 180, # H1
|
||||
'09': 270, # Q1-Q3
|
||||
'12': 360 # 全年
|
||||
}.get(quarter, 360)
|
||||
|
||||
#循环调用类
|
||||
def get_finance_data_range( ts_code, start_date=START_DATE, end_date=END_DATE,TS_TOKEN=TS_TOKEN):
|
||||
"""
|
||||
获取指定日期范围内的所有财报数据
|
||||
:param ts_code: 股票代码
|
||||
:param start_date: 开始日期 (yyyyMMdd)
|
||||
:param end_date: 结束日期 (yyyyMMdd)
|
||||
:return: 合并后的财报数据列表
|
||||
"""
|
||||
results = pd.DataFrame()
|
||||
|
||||
# 生成所有可能的财报日期 (季度末)
|
||||
years = range(int(start_date[:4]), int(end_date[:4]) + 1)
|
||||
report_dates = []
|
||||
for year in years:
|
||||
report_dates.extend([
|
||||
f"{year}0331", # Q1
|
||||
f"{year}0630", # Q2
|
||||
f"{year}0930", # Q3
|
||||
f"{year}1231" # Q4
|
||||
])
|
||||
|
||||
# 筛选在指定日期范围内的财报日期
|
||||
report_dates = [
|
||||
date for date in report_dates
|
||||
if start_date <= date <= end_date
|
||||
]
|
||||
report_dates=get_released_report_dates(start_date,end_date)
|
||||
|
||||
# 按日期顺序获取财报数据
|
||||
analyzer = FinanceData(TS_TOKEN)
|
||||
analyzer.ts_code=tscodeCheck(ts_code)
|
||||
for fin_date in sorted(report_dates):
|
||||
analyzer.fin_date = fin_date
|
||||
try:
|
||||
data = analyzer.get_finance_data()
|
||||
results = pd.concat([pd.DataFrame(results), data.to_frame().T], ignore_index=True) if len(results) > 0 else data.to_frame().T
|
||||
except Exception as e:
|
||||
print(f"获取 {ts_code} {fin_date} 财报数据失败: {str(e)}")
|
||||
continue
|
||||
|
||||
return results
|
||||
# 使用示例
|
||||
if __name__ == "__main__":
|
||||
# 导入当站目录的config文件
|
||||
try:
|
||||
# 尝试相对导入（作为包的一部分）
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
except (ImportError, SystemError):
|
||||
# 失败则使用绝对导入（直接运行脚本）
|
||||
from config import TS_TOKEN, START_DATE, END_DATE, PRECISION_CONFIG
|
||||
#token = "your_tushare_token" # 替换为实际token
|
||||
#analyzer = FinanceData(TS_TOKEN)
|
||||
|
||||
# 获取002273.SZ在2022Q1的财务数据
|
||||
#analyzer.ts_code = '300316.SZ'
|
||||
#analyzer.fin_date = '20240630'
|
||||
|
||||
|
||||
#result = analyzer.get_finance_data()
|
||||
|
||||
result=get_finance_data_range("300316",start_date='20200101',end_date='2025060')
|
||||
print(result)
|
||||
#print(pd.Series(result))
|
||||
@@ -0,0 +1,70 @@
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from .data_source import get_tushare_pro
|
||||
|
||||
# 导入当站目录的config文件
|
||||
try:
|
||||
# 尝试相对导入（作为包的一部分）
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE
|
||||
from .stock_utils import dataCorrect
|
||||
except (ImportError, SystemError):
|
||||
# 失败则使用绝对导入（直接运行脚本）
|
||||
from config import TS_TOKEN, START_DATE, END_DATE
|
||||
from stock_utils import dataCorrect
|
||||
|
||||
def getStockParam(TS_CODE,START_DATE=START_DATE,END_DATE=END_DATE):
|
||||
pro = get_tushare_pro()
|
||||
"""
|
||||
从tushare daily_basic接口获取单只股票的所有基础数据,并进行数据修正。
|
||||
|
||||
Parameters:
|
||||
TS_CODE (str): 股票代码,格式为 '股票代码.SZ' 或 '股票代码.SH',例如 '000001.SZ'
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含股票基础数据的DataFrame,字段说明如下：
|
||||
ts_code: 股票代码
|
||||
trade_date: 交易日期
|
||||
close: 收盘价
|
||||
turnover_rate: 换手率（%）
|
||||
turnover_rate_f: 换手率（自由流通股）
|
||||
volume_ratio: 量比
|
||||
pe: 市盈率（总市值/净利润,亏损的PE为空）
|
||||
pe_ttm: 市盈率（TTM,亏损的PE为空）
|
||||
pb: 市净率（总市值/净资产）
|
||||
ps: 市销率（总市值/营业收入）
|
||||
ps_ttm: 市销率（TTM）
|
||||
dv_ratio: 股息率（%）
|
||||
dv_ttm: 股息率（TTM）
|
||||
total_share: 总股本（万股）
|
||||
float_share: 流通股本（万股）
|
||||
free_share: 自由流通股本（万股）
|
||||
total_mv: 总市值（万元）
|
||||
circ_mv: 流通市值（万元）
|
||||
|
||||
Raises:
|
||||
Exception: 如果从Tushare接口获取数据时发生错误。
|
||||
"""
|
||||
try:
|
||||
# 获取股票基础数据
|
||||
df = pro.daily_basic(ts_code=TS_CODE, start_date=START_DATE, end_date=END_DATE)
|
||||
|
||||
# 需要修正的列
|
||||
cols = ['close', 'turnover_rate', 'turnover_rate_f', 'volume_ratio', 'pe', 'pe_ttm',
|
||||
'pb', 'ps', 'ps_ttm', 'dv_ratio', 'dv_ttm', 'total_share', 'float_share',
|
||||
'free_share', 'total_mv', 'circ_mv']
|
||||
|
||||
# 调用dataCorrect函数进行数据修正
|
||||
df = dataCorrect(df, cols)
|
||||
|
||||
return df
|
||||
except Exception as e:
|
||||
# 捕获并处理异常
|
||||
print(f"错误: 获取股票 {TS_CODE} 的基础数据时发生错误: {e}")
|
||||
return pd.DataFrame()
|
||||
if __name__ == "__main__":
|
||||
# 简单测试
|
||||
import datetime
|
||||
yesterday = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime("%Y%m%d")
|
||||
test_df = getStockParam("601398.SH", START_DATE=yesterday, END_DATE=yesterday)
|
||||
print(test_df)
|
||||
@@ -0,0 +1 @@
|
||||
from ..utils.mysql_handler import MySQLDB # noqa: F401 — 向后兼容重新导出
|
||||
@@ -0,0 +1,15 @@
|
||||
# 扫描配置
|
||||
|
||||
# 指定行业板块
|
||||
INDUSTRIES = ["软件服务", "运输设备", "电气设备", "元器件", "火力发电",
|
||||
"医药商业", "汽车配件", "新型电力", "铅锌", "通信设备", "IT设备",
|
||||
"工程机械", "证券", "生物制药", "百货", "食品", "机械基件",
|
||||
"汽车整车", "煤炭开采", "白酒", "铝", "铜", "小金属",
|
||||
"互联网", "航空", "超市连锁", "轻工机械", "电器仪表", "半导体",
|
||||
"公共交通", "电信运营"]
|
||||
|
||||
# 判断阈值
|
||||
PRICE_VOLATILITY_THRESHOLD = 5 # 价格波动幅度均值上限（百分比）
|
||||
MA20_STD_THRESHOLD = 0.05 # 移动平均线标准差上限（相对于均值的百分比）
|
||||
ATR_THRESHOLD = 0.02 # ATR均值上限（相对于收盘价均值的百分比）
|
||||
BOLLINGER_BAND_WIDTH_THRESHOLD = 5 # 布林带宽度均值上限（百分比）
|
||||
@@ -0,0 +1,190 @@
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
def smooth_series_brush(series: pd.Series, window_size: int = 7, threshold_factor: float = 0.5, max_brush_length: int = 5) -> pd.Series:
|
||||
"""
|
||||
平滑处理pandas序列中的连续毛刺数据，使用前值或后值填充。
|
||||
|
||||
参数:
|
||||
series (pd.Series): 输入的pandas序列。
|
||||
window_size (int): 用于检测毛刺的滑动窗口大小。必须为奇数。默认为7。
|
||||
threshold_factor (float): 判断毛刺的阈值因子。如果 abs(value - median) / median > threshold_factor，则认为是毛刺。默认为0.5 (50%)。
|
||||
max_brush_length (int): 允许的最大连续毛刺长度。超过此长度的连续点将不被处理。默认为5。
|
||||
|
||||
返回:
|
||||
pd.Series: 处理后的平滑序列。
|
||||
"""
|
||||
if not isinstance(series, pd.Series):
|
||||
raise TypeError("输入必须是 pandas Series 对象。")
|
||||
|
||||
if window_size % 2 == 0:
|
||||
raise ValueError("window_size 必须是奇数。")
|
||||
|
||||
# 创建副本以避免修改原始数据
|
||||
smoothed_series = series.copy()
|
||||
|
||||
# 用于标记是否为毛刺的布尔序列
|
||||
is_brush = pd.Series([False] * len(series), index=series.index)
|
||||
|
||||
half_window = window_size // 2
|
||||
|
||||
# --- 第一步：检测毛刺 ---
|
||||
for i in range(len(series)):
|
||||
start_idx = max(0, i - half_window)
|
||||
end_idx = min(len(series), i + half_window + 1)
|
||||
|
||||
# 获取当前窗口数据
|
||||
window_data = series.iloc[start_idx:end_idx]
|
||||
|
||||
if len(window_data) < 2:
|
||||
continue
|
||||
|
||||
# 计算窗口中位数
|
||||
window_median = window_data.median()
|
||||
|
||||
# 避免除以零
|
||||
if window_median == 0:
|
||||
continue
|
||||
|
||||
current_value = series.iloc[i]
|
||||
|
||||
# 计算偏差比例
|
||||
deviation_ratio = abs(current_value - window_median) / abs(window_median)
|
||||
|
||||
# 如果偏差超过阈值，则标记为毛刺
|
||||
if deviation_ratio > threshold_factor:
|
||||
is_brush.iloc[i] = True
|
||||
|
||||
# --- 第二步：处理连续的毛刺段 ---
|
||||
# 使用 cumsum 技巧识别连续毛刺段
|
||||
brush_groups = (is_brush != is_brush.shift()).cumsum() * is_brush
|
||||
|
||||
# 遍历每个被标记为毛刺的组
|
||||
for group_id in brush_groups[brush_groups != 0].unique():
|
||||
if pd.isna(group_id):
|
||||
continue
|
||||
|
||||
brush_indices = brush_groups[brush_groups == group_id].index
|
||||
|
||||
# 检查连续毛刺长度
|
||||
if len(brush_indices) > max_brush_length:
|
||||
print(f"警告: 发现长度为 {len(brush_indices)} 的连续毛刺段 (超过 max_brush_length={max_brush_length})，将不进行平滑处理。")
|
||||
continue
|
||||
|
||||
# --- 平滑处理：使用前值或后值填充 ---
|
||||
# 查找前一个非毛刺点
|
||||
prev_valid_val = None
|
||||
start_loc = series.index.get_loc(brush_indices[0])
|
||||
for j in range(start_loc - 1, -1, -1):
|
||||
if not is_brush.iloc[j]:
|
||||
prev_valid_val = series.iloc[j]
|
||||
break
|
||||
|
||||
# 查找后一个非毛刺点
|
||||
next_valid_val = None
|
||||
end_loc = series.index.get_loc(brush_indices[-1])
|
||||
for j in range(end_loc + 1, len(series)):
|
||||
if not is_brush.iloc[j]:
|
||||
next_valid_val = series.iloc[j]
|
||||
break
|
||||
|
||||
# 决定使用哪个值填充
|
||||
if prev_valid_val is not None:
|
||||
fill_value = next_valid_val
|
||||
elif next_valid_val is not None:
|
||||
fill_value = prev_valid_val
|
||||
else:
|
||||
print(f"警告: 毛刺段 {brush_indices} 无有效邻居，使用全局中位数填充。")
|
||||
fill_value = series.median()
|
||||
|
||||
# 用 fill_value 填充整个毛刺段
|
||||
for idx in brush_indices:
|
||||
smoothed_series.loc[idx] = fill_value
|
||||
|
||||
return smoothed_series
|
||||
|
||||
|
||||
def smooth_dataframe_brush(df: pd.DataFrame, target_columns: list, **kwargs) -> pd.DataFrame:
|
||||
"""
|
||||
对DataFrame中的指定列进行毛刺平滑处理。
|
||||
|
||||
参数:
|
||||
df (pd.DataFrame): 输入的pandas DataFrame。
|
||||
target_columns (list): 需要去毛刺处理的列名列表。
|
||||
**kwargs: 传递给 smooth_series_brush 函数的参数 (如 window_size, threshold_factor, max_brush_length)。
|
||||
|
||||
返回:
|
||||
pd.DataFrame: 处理后的DataFrame，指定列已平滑，其余列不变。
|
||||
"""
|
||||
if not isinstance(df, pd.DataFrame):
|
||||
raise TypeError("输入必须是 pandas DataFrame 对象。")
|
||||
|
||||
# 创建副本以避免修改原始数据
|
||||
result_df = df.copy()
|
||||
|
||||
# 检查目标列是否都存在于DataFrame中
|
||||
missing_cols = [col for col in target_columns if col not in df.columns]
|
||||
if missing_cols:
|
||||
raise ValueError(f"以下列不在DataFrame中: {missing_cols}")
|
||||
|
||||
# 对每个目标列应用平滑函数
|
||||
for col in target_columns:
|
||||
print(f"正在处理列: {col}")
|
||||
try:
|
||||
# 应用去毛刺函数
|
||||
result_df[col] = smooth_series_brush(df[col], **kwargs)
|
||||
except Exception as e:
|
||||
print(f"处理列 {col} 时出错: {e}")
|
||||
# 可以选择保留原始数据或抛出异常
|
||||
# 这里选择保留原始数据
|
||||
continue
|
||||
|
||||
return result_df
|
||||
|
||||
|
||||
# --- 示例 ---
|
||||
if __name__ == "__main__":
|
||||
# 1. 创建示例 DataFrame
|
||||
dates = pd.date_range('2023-01-01', periods=20, freq='D')
|
||||
# 需要处理的列
|
||||
values_to_smooth = [10, 11, 10.5, 12, 11.8, 50, 12.1, 11.9, 10, 10.2,
|
||||
9.8, 100, 105, 99, 10.1, 9.9, 10.3, 5, 10.2, 10.1]
|
||||
# 不需要处理的列 (例如，另一个传感器数据)
|
||||
other_data = np.random.randn(20).cumsum() + 100 # 累积和，模拟趋势
|
||||
|
||||
# 构建 DataFrame
|
||||
df_original = pd.DataFrame({
|
||||
'Date': dates,
|
||||
'Sensor_A': values_to_smooth, # 需要去毛刺
|
||||
'Sensor_B': other_data, # 不需要处理
|
||||
'Other_Info': range(20) # 其他信息，不需要处理
|
||||
})
|
||||
# 设置日期为索引 (常见做法)
|
||||
df_original.set_index('Date', inplace=True)
|
||||
|
||||
print("原始 DataFrame:")
|
||||
print(df_original.head(10))
|
||||
print("\n" + "="*50 + "\n")
|
||||
|
||||
# 2. 应用平滑函数
|
||||
# 指定需要处理的列
|
||||
columns_to_smooth = ['Sensor_A']
|
||||
# 调用新函数
|
||||
df_smoothed = smooth_dataframe_brush(
|
||||
df_original,
|
||||
target_columns=columns_to_smooth,
|
||||
window_size=5,
|
||||
threshold_factor=0.3,
|
||||
max_brush_length=5
|
||||
)
|
||||
|
||||
print("平滑后的 DataFrame:")
|
||||
print(df_smoothed.head(10))
|
||||
print("\n" + "="*50 + "\n")
|
||||
|
||||
# 3. 比较
|
||||
comparison_df = df_original.copy()
|
||||
comparison_df['Sensor_A_Smoothed'] = df_smoothed['Sensor_A']
|
||||
comparison_df['Difference'] = comparison_df['Sensor_A'] - comparison_df['Sensor_A_Smoothed']
|
||||
print("对比 (原始 Sensor_A vs 平滑后 vs 差异):")
|
||||
print(comparison_df[['Sensor_A', 'Sensor_A_Smoothed', 'Difference']].head(10))
|
||||
@@ -0,0 +1,125 @@
|
||||
import pandas as pd
|
||||
# 将项目根目录添加到 sys.path
|
||||
#project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
#sys.path.append(project_root)
|
||||
|
||||
# 导入当站目录的config文件
|
||||
try:
|
||||
# 尝试相对导入（作为包的一部分）
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE
|
||||
from .stock_utils import dataCorrect,get_trading_dates,tscodeCheck
|
||||
except (ImportError, SystemError):
|
||||
# 失败则使用绝对导入（直接运行脚本）
|
||||
from config import TS_TOKEN, START_DATE, END_DATE
|
||||
from stock_utils import dataCorrect,get_trading_dates,tscodeCheck
|
||||
|
||||
from .data_source import get_tushare_pro
|
||||
pro = get_tushare_pro()
|
||||
|
||||
def getStockMargin(tscode: str, start_date: str=START_DATE, end_date: str=END_DATE) -> pd.DataFrame:
|
||||
"""
|
||||
获取指定股票代码在时间区间内的每日融资明细数据
|
||||
|
||||
参数:
|
||||
tscode (str): 股票代码，格式如 '600000.SH'
|
||||
start_date (str): 开始日期，格式 'YYYY-MM-DD'
|
||||
end_date (str): 结束日期，格式 'YYYY-MM-DD'
|
||||
|
||||
返回:
|
||||
pd.DataFrame: 包含融资明细数据的DataFrame，列包括:
|
||||
- trade_date: 交易日期
|
||||
- tscode: 股票代码
|
||||
- rzye: 融资余额(元)
|
||||
- rqye float 融券余额(元)
|
||||
- rzmre float 融资买入额(元)
|
||||
- rqyl float 融券余量（股）
|
||||
- rzche float 融资偿还额(元)
|
||||
- rqchl float 融券偿还量(股)
|
||||
- rqmcl float 融券卖出量(股,份,手)
|
||||
- rzrqye float 融资融券余额(元)
|
||||
|
||||
异常处理:
|
||||
- 若tushare接口调用失败，打印错误信息并返回空DataFrame
|
||||
"""
|
||||
try:
|
||||
tscode=tscodeCheck(tscode)
|
||||
# 调用tushare接口
|
||||
df = pro.margin_detail(ts_code=tscode, start_date=start_date, end_date=end_date)
|
||||
cols=["rzye","rqye","rzmre","rqyl","rzche","rqchl","rqmcl","rzrqye"]
|
||||
df = dataCorrect(df,cols)
|
||||
if df.empty:
|
||||
print(f"未找到{tscode}在{start_date}至{end_date}期间的融资数据")
|
||||
return df
|
||||
except Exception as e:
|
||||
print(f"获取融资数据失败: {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
def getDailyMargin(trade_date: str = None, start_date: str = None, end_date: str = None, exchange_id: str = None) -> pd.DataFrame:
|
||||
"""
|
||||
获取指定日期或时间区间内的每日融资明细数据
|
||||
|
||||
参数:
|
||||
trade_date (str, optional): 指定单个交易日，格式 'YYYY-MM-DD'。与start_date/end_date互斥
|
||||
start_date (str, optional): 开始日期，格式 'YYYY-MM-DD'。需与end_date同时使用
|
||||
end_date (str, optional): 结束日期，格式 'YYYY-MM-DD'。需与start_date同时使用
|
||||
exchange_id (str, optional): 交易所代码，如 'SSE'（上交所）、'SZSE'（深交所）
|
||||
|
||||
返回:
|
||||
pd.DataFrame: 包含每日融资明细数据的DataFrame，列包括:
|
||||
- exchange_id: 交易所代码
|
||||
- trade_date: 交易日期
|
||||
- rzye: 融资余额(元)
|
||||
- rqye: 融券余额(元)
|
||||
- rzmre: 融资买入额(元)
|
||||
- rqyl: 融券余量（股）
|
||||
- rzche: 融资偿还额(元)
|
||||
- rqchl: 融券偿还量(股)
|
||||
- rqmcl: 融券卖出量(股,份,手)
|
||||
- rzrqye: 融资融券余额(元)
|
||||
|
||||
异常处理:
|
||||
- 若参数组合无效，打印错误信息并返回空DataFrame
|
||||
- 若tushare接口调用失败，打印错误信息并返回空DataFrame
|
||||
"""
|
||||
try:
|
||||
# 日期格式转换
|
||||
if trade_date:
|
||||
trade_date = trade_date.replace("-", "")
|
||||
if start_date:
|
||||
start_date = start_date.replace("-", "")
|
||||
if end_date:
|
||||
end_date = end_date.replace("-", "")
|
||||
# 参数校验
|
||||
if trade_date and (start_date or end_date):
|
||||
print("错误：trade_date不能与start_date/end_date同时使用")
|
||||
return pd.DataFrame()
|
||||
|
||||
if (start_date and not end_date) or (end_date and not start_date):
|
||||
print("错误：start_date和end_date必须同时使用")
|
||||
return pd.DataFrame()
|
||||
|
||||
# 调用tushare接口
|
||||
df = pro.margin(exchange_id=exchange_id,
|
||||
trade_date=trade_date,
|
||||
start_date=start_date,
|
||||
end_date=end_date)
|
||||
|
||||
if df.empty:
|
||||
print("未找到符合条件的融资数据")
|
||||
return df
|
||||
except Exception as e:
|
||||
print(f"获取每日融资数据失败: {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试getStockMargin函数
|
||||
test_code = "600000.SH"
|
||||
test_start = "20150101"
|
||||
test_end = "20250731"
|
||||
|
||||
result = getStockMargin(test_code, test_start, test_end)
|
||||
print("返回结果示例:")
|
||||
print(result.head())
|
||||
|
||||
if not result.empty:
|
||||
print(f"\n返回数据行数: {len(result)}")
|
||||
@@ -0,0 +1,163 @@
|
||||
import pandas as pd
|
||||
# 将项目根目录添加到 sys.path
|
||||
#project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
#sys.path.append(project_root)
|
||||
|
||||
# 导入当站目录的config文件
|
||||
try:
|
||||
# 尝试相对导入（作为包的一部分）
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE
|
||||
from .stock_utils import dataCorrect,get_trading_dates,tscodeCheck
|
||||
except (ImportError, SystemError):
|
||||
# 失败则使用绝对导入（直接运行脚本）
|
||||
from config import TS_TOKEN, START_DATE, END_DATE
|
||||
from stock_utils import dataCorrect,get_trading_dates,tscodeCheck
|
||||
|
||||
from .data_source import get_tushare_pro
|
||||
pro = get_tushare_pro()
|
||||
|
||||
def getStockBasic(TS_CODE,START_DATE=START_DATE,END_DATE=END_DATE):
|
||||
"""
|
||||
从tushare daily接口获取单只股票的所有返回数据,并进行数据修正。
|
||||
|
||||
Parameters:
|
||||
TS_CODE (str): 股票代码,格式为 '股票代码.SZ' 或 '股票代码.SH',例如 '000001.SZ'
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含股票日线数据的DataFrame,字段说明如下：
|
||||
ts_code: 股票代码
|
||||
trade_date: 交易日期
|
||||
open: 开盘价
|
||||
high: 最高价
|
||||
low: 最低价
|
||||
close: 收盘价
|
||||
pre_close: 前日收盘价
|
||||
change: 涨跌额
|
||||
pct_chg: 涨跌幅（百分比）
|
||||
vol: 成交量（手）
|
||||
amount: 成交额（千元）
|
||||
|
||||
Raises:
|
||||
Exception: 如果从Tushare接口获取数据时发生错误。
|
||||
"""
|
||||
try:
|
||||
# 获取股票日线数据
|
||||
df = pro.daily(ts_code=TS_CODE, start_date=START_DATE, end_date=END_DATE)
|
||||
|
||||
# 需要修正的列
|
||||
cols = ['open', 'high', 'low', 'close', 'pre_close', 'change', 'pct_chg', 'vol', 'amount']
|
||||
|
||||
# 调用dataCorrect函数进行数据修正
|
||||
df = dataCorrect(df, cols)
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
# 捕获并处理异常
|
||||
print(f"错误: 获取股票 {TS_CODE} 的日线数据时发生错误: {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
def getStockInfo(TS_CODE):
|
||||
"""
|
||||
从tushare的stock_basic接口获取单只股票的基本信息。
|
||||
|
||||
Parameters:
|
||||
TS_CODE (str): 股票代码,格式为 '股票代码.SZ' 或 '股票代码.SH',例如 '000001.SZ'
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含股票基本信息的DataFrame,字段说明如下：
|
||||
ts_code: 股票代码
|
||||
symbol: 股票代码（不带后缀）
|
||||
name: 股票名称
|
||||
area: 所在地域
|
||||
industry: 所属行业
|
||||
market: 市场类型（主板/创业板/科创板等）
|
||||
list_date: 上市日期
|
||||
fullname: 股票全称
|
||||
enname: 英文全称
|
||||
exchange: 交易所代码
|
||||
curr_type: 交易货币
|
||||
list_status: 上市状态
|
||||
is_hs: 是否沪深港通标的
|
||||
|
||||
Raises:
|
||||
Exception: 如果从Tushare接口获取数据时发生错误。
|
||||
"""
|
||||
try:
|
||||
TS_CODE=tscodeCheck(TS_CODE)
|
||||
# 调用stock_basic接口获取股票基本信息
|
||||
df = pro.stock_basic(ts_code=TS_CODE)
|
||||
|
||||
# 如果没有获取到数据，返回空的DataFrame
|
||||
if df.empty:
|
||||
print(f"警告: 未找到股票 {TS_CODE} 的基本信息")
|
||||
return pd.DataFrame()
|
||||
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
# 捕获并处理异常
|
||||
print(f"错误: 获取股票 {TS_CODE} 的基本信息时发生错误: {e}")
|
||||
return pd.DataFrame()
|
||||
|
||||
def getStockListByIndustry(industry):
|
||||
"""
|
||||
根据行业名称获取股票列表。
|
||||
|
||||
该函数首先通过 `index_classify` 接口获取行业分类的级别和行业代码,
|
||||
然后使用 `index_member_all` 接口获取该行业下的所有股票列表。
|
||||
|
||||
Parameters:
|
||||
industry (str): 行业名称,例如 "银行"、"医药" 等。
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含股票代码 (ts_code) 和股票名称 (name) 的 DataFrame。
|
||||
如果未找到匹配的行业或股票,返回空的 DataFrame。
|
||||
|
||||
Raises:
|
||||
Exception: 如果从 Tushare 接口获取数据时发生错误。
|
||||
"""
|
||||
try:
|
||||
# 1. 获取行业分类信息
|
||||
# 使用 index_classify 接口获取所有行业分类信息
|
||||
industry_df = pro.index_classify(level='', src='SW2021')
|
||||
|
||||
# 过滤出与给定行业名称匹配的行业
|
||||
industry_info = industry_df[industry_df['industry_name'] == industry]
|
||||
|
||||
# 如果没有找到匹配的行业,返回空的 DataFrame
|
||||
if industry_info.empty:
|
||||
print(f"警告: 未找到行业 '{industry}' 的分类信息")
|
||||
return pd.DataFrame(columns=['ts_code', 'name'])
|
||||
|
||||
# 获取行业代码和级别
|
||||
industry_code = industry_info.iloc[0]['index_code']
|
||||
industry_level = industry_info.iloc[0]['level']
|
||||
print(industry_info)
|
||||
# 2. 根据行业级别调用 index_member_all 接口
|
||||
if industry_level == 'L1':
|
||||
stock_list_df = pro.index_member_all(l1_code=industry_code)
|
||||
elif industry_level == 'L2':
|
||||
stock_list_df = pro.index_member_all(l2_code=industry_code)
|
||||
elif industry_level == 'L3':
|
||||
stock_list_df = pro.index_member_all(l3_code=industry_code)
|
||||
else:
|
||||
print(f"警告: 未知的行业级别 '{industry_level}'")
|
||||
return pd.DataFrame(columns=['ts_code', 'name'])
|
||||
|
||||
# 如果没有找到股票,返回空的 DataFrame
|
||||
if stock_list_df.empty:
|
||||
print(f"警告: 行业 '{industry}' 下没有找到股票")
|
||||
return pd.DataFrame(columns=['ts_code', 'name'])
|
||||
|
||||
# 3. 返回股票代码和名称
|
||||
return stock_list_df[['ts_code', 'name']]
|
||||
|
||||
except Exception as e:
|
||||
# 捕获并处理异常
|
||||
print(f"错误: 获取行业 '{industry}' 的股票列表时发生错误: {e}")
|
||||
return pd.DataFrame(columns=['ts_code', 'name'])
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
df = getStockListByIndustry('果蔬加工')
|
||||
print(df)
|
||||
@@ -0,0 +1,363 @@
|
||||
import pandas as pd
|
||||
from django.http import HttpResponse
|
||||
from rest_framework.response import Response
|
||||
try:
|
||||
from .config import TS_TOKEN, START_DATE, END_DATE
|
||||
except (ImportError, SystemError):
|
||||
from config import TS_TOKEN, START_DATE, END_DATE
|
||||
|
||||
from .data_source import get_tushare_pro
|
||||
|
||||
# 统一入口（全局单例，向后兼容旧代码直接访问 pro）
|
||||
pro = get_tushare_pro()
|
||||
|
||||
def dataCorrect(df, columns=None):
|
||||
"""
|
||||
检查并修正DataFrame中的NaN或None值。
|
||||
1. 如果前后有数字,该字段取前后值的均值填入
|
||||
2. 如果仅前或后有数字,该字段取copy前或后数字
|
||||
3. 若前后都为NaN,则设为0
|
||||
4. 若出发点修改,打印修改情况到屏幕
|
||||
|
||||
Parameters:
|
||||
df (pd.DataFrame): 要处理的数据框
|
||||
columns (list): 要检查的列名列表,如果为None则检查所有列
|
||||
"""
|
||||
# 如果没有指定列,则检查所有列
|
||||
if columns is None:
|
||||
columns = df.columns
|
||||
# 遍历指定列
|
||||
for col in columns:
|
||||
# 遍历每一行
|
||||
for i in range(len(df)):
|
||||
# 检查当前值是否为NaN或None
|
||||
if pd.isna(df.at[i, col]):
|
||||
# 获取前后值
|
||||
prev_val = df.at[i-1, col] if i > 0 else None
|
||||
next_val = df.at[i+1, col] if i < len(df) - 1 else None
|
||||
|
||||
# 检查前后值是否为有效数字
|
||||
prev_valid = prev_val is not None and not pd.isna(prev_val)
|
||||
next_valid = next_val is not None and not pd.isna(next_val)
|
||||
# 如果前后都有值,取均值
|
||||
if prev_valid and next_valid:
|
||||
new_val = (prev_val + next_val) / 2
|
||||
df.at[i, col] = new_val
|
||||
#print(f"修正 {col} 列第 {i} 行: 前后值均值填充为 {new_val}")
|
||||
|
||||
# 如果只有前值,取前值
|
||||
elif prev_valid:
|
||||
df.at[i, col] = prev_val
|
||||
#print(f"修正 {col} 列第 {i} 行: 前值填充为 {prev_val}")
|
||||
|
||||
# 如果只有后值,取后值
|
||||
elif next_valid:
|
||||
df.at[i, col] = next_val
|
||||
#print(f"修正 {col} 列第 {i} 行: 后值填充为 {next_val}")
|
||||
|
||||
# 如果前后都没有有效值,设为0
|
||||
else:
|
||||
df.at[i, col] = 0
|
||||
#print(f"修正 {col} 列第 {i} 行: 前后均无有效值,设为0")
|
||||
|
||||
return df
|
||||
|
||||
def dataMerge(*dfs):
|
||||
"""
|
||||
合并多个股票数据DataFrame，确保合并后的数据ts_code和trade_date匹配。
|
||||
|
||||
Parameters:
|
||||
*dfs: 可变数量的DataFrame参数，每个DataFrame应包含ts_code和trade_date列
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 合并后的DataFrame，包含所有输入DataFrame的列。如果某行的ts_code或trade_date不匹配，则用NaN填充。
|
||||
"""
|
||||
if len(dfs) < 2:
|
||||
raise ValueError("至少需要提供2个DataFrame进行合并")
|
||||
|
||||
# 检查所有DataFrame是否都有ts_code和trade_date列
|
||||
for df in dfs:
|
||||
if 'ts_code' not in df.columns or 'trade_date' not in df.columns:
|
||||
raise ValueError("所有DataFrame必须包含ts_code和trade_date列")
|
||||
|
||||
# 初始化合并结果为第一个DataFrame
|
||||
merged_df = dfs[0].copy()
|
||||
|
||||
# 逐个合并剩余的DataFrame
|
||||
for df in dfs[1:]:
|
||||
merged_df = pd.merge(merged_df, df, on=['ts_code', 'trade_date'], how='outer')
|
||||
|
||||
return merged_df
|
||||
|
||||
def tscodeCheck(tscode):
|
||||
"""
|
||||
检查并修正股票代码格式。
|
||||
|
||||
Parameters:
|
||||
tscode (str): 股票代码
|
||||
|
||||
Returns:
|
||||
str: 格式正确的股票代码 (如 '000001.SZ')
|
||||
|
||||
Raises:
|
||||
ValueError: 如果输入不符合要求
|
||||
"""
|
||||
if not isinstance(tscode, str):
|
||||
raise ValueError("输入必须是字符串")
|
||||
|
||||
tscode = tscode.upper() # 转为大写
|
||||
|
||||
# 检查长度
|
||||
if len(tscode) < 6:
|
||||
raise ValueError("股票代码长度不能小于6位")
|
||||
elif len(tscode) == 6:
|
||||
if not tscode.isdigit():
|
||||
raise ValueError("6位股票代码必须全为数字")
|
||||
# 根据开头添加后缀
|
||||
if tscode.startswith('00') or tscode.startswith('3'):
|
||||
return f"{tscode}.SZ"
|
||||
elif tscode.startswith(('60','68')): # 68为科创板
|
||||
return f"{tscode}.SH"
|
||||
elif tscode.startswith(('8', '9')):
|
||||
return f"{tscode}.BJ"
|
||||
else:
|
||||
raise ValueError("未知的6位股票代码开头")
|
||||
elif len(tscode) == 9:
|
||||
prefix = tscode[:6]
|
||||
suffix = tscode[-3:]
|
||||
if not prefix.isdigit():
|
||||
raise ValueError("9位股票代码前6位必须为数字")
|
||||
if suffix not in ('.SZ', '.SH', '.BJ'):
|
||||
raise ValueError("9位股票代码后缀必须是.SZ/.SH/.BJ")
|
||||
return tscode
|
||||
else:
|
||||
raise ValueError("股票代码长度必须为6位或9位")
|
||||
|
||||
|
||||
|
||||
def get_trading_dates(trade_date):
|
||||
"""
|
||||
获取上一个季度财报日期到给定日期的所有交易日期
|
||||
|
||||
Parameters:
|
||||
trade_date (str): 给定日期 (格式yyyymmdd)
|
||||
Returns:
|
||||
list: 交易日期列表 (格式yyyymmdd)
|
||||
"""
|
||||
# 将输入日期转为datetime
|
||||
try:
|
||||
current_date = pd.to_datetime(trade_date, format='%Y%m%d')
|
||||
except:
|
||||
raise ValueError("trade_date格式应为yyyymmdd")
|
||||
|
||||
# 计算上一个季度财报日期 (3/31, 6/30, 9/30, 12/31)
|
||||
year = current_date.year
|
||||
month = current_date.month
|
||||
if month < 4:
|
||||
prev_report_date = pd.Timestamp(year-1, 12, 31)
|
||||
elif month < 7:
|
||||
prev_report_date = pd.Timestamp(year, 3, 31)
|
||||
elif month < 10:
|
||||
prev_report_date = pd.Timestamp(year, 6, 30)
|
||||
else:
|
||||
prev_report_date = pd.Timestamp(year, 9, 30)
|
||||
|
||||
# 获取交易日历
|
||||
calendar_df = pro.trade_cal(exchange='', start_date=prev_report_date.strftime('%Y%m%d'), end_date=trade_date)
|
||||
|
||||
# 过滤交易日历
|
||||
calendar_df['cal_date_dt'] = pd.to_datetime(calendar_df['cal_date'], format='%Y%m%d')
|
||||
filtered_dates = calendar_df[
|
||||
(calendar_df['cal_date_dt'] > prev_report_date) &
|
||||
(calendar_df['cal_date_dt'] <= current_date) &
|
||||
(calendar_df['is_open'] == 1)
|
||||
]
|
||||
|
||||
# 返回日期列表 (格式yyyymmdd)
|
||||
return filtered_dates['cal_date'].tolist()
|
||||
|
||||
def get_next_report_date(trade_date):
|
||||
"""
|
||||
获取给定日期的下一个财报日期
|
||||
|
||||
Parameters:
|
||||
trade_date (str): 给定日期 (格式yyyymmdd)
|
||||
|
||||
Returns:
|
||||
str: 下一个财报日期 (格式yyyymmdd)
|
||||
"""
|
||||
try:
|
||||
current_date = pd.to_datetime(trade_date, format='%Y%m%d')
|
||||
except:
|
||||
raise ValueError("trade_date格式应为yyyymmdd")
|
||||
|
||||
year = current_date.year
|
||||
month = current_date.month
|
||||
day = current_date.day
|
||||
|
||||
if month < 3 or (month == 3 and day < 31):
|
||||
return f"{year}0331"
|
||||
elif month < 6 or (month == 6 and day < 30):
|
||||
return f"{year}0630"
|
||||
elif month < 9 or (month == 9 and day < 30):
|
||||
return f"{year}0930"
|
||||
elif month < 12 or (month == 12 and day < 31):
|
||||
return f"{year}1231"
|
||||
else:
|
||||
return f"{year+1}0331"
|
||||
|
||||
def viewFunc_singleParam(request, data_func, param_name='tscode', default_value=None):
|
||||
"""
|
||||
通用单参数视图包装器。
|
||||
|
||||
:param request: Django request 对象
|
||||
:param data_func: 接收单个参数的数据处理函数
|
||||
:param param_name: URL 查询参数名
|
||||
:param default_value: 参数默认值
|
||||
:return: Response
|
||||
"""
|
||||
param = request.GET.get(param_name, default_value)
|
||||
if not param:
|
||||
return Response({'error': f'缺少 {param_name} 参数'}, status=400)
|
||||
|
||||
try:
|
||||
data = data_func(param)
|
||||
dict_data = data.to_dict(orient='records') if isinstance(data, pd.DataFrame) else data
|
||||
return Response(dict_data)
|
||||
except ImportError:
|
||||
return Response({'error': f'模块不存在'}, status=500)
|
||||
except Exception as e:
|
||||
return Response({'error': str(e)}, status=500)
|
||||
|
||||
|
||||
def viewFunc_tsCodeAndDate(request,data_func):
|
||||
"""
|
||||
通用数据处理函数
|
||||
:param request: Django request对象
|
||||
:param data_func: 数据处理函数（需返回Response）
|
||||
:return: Response
|
||||
"""
|
||||
#通过url 传入 tscode 变量
|
||||
tscode = request.GET.get('tscode','000001.SZ') # 从URL获取industry参数
|
||||
tscode = tscodeCheck(tscode)
|
||||
if not tscode:
|
||||
return Response({'error': '缺少 tscode 参数'}, status=400)
|
||||
|
||||
start_date = request.GET.get('start_date', START_DATE)
|
||||
end_date = request.GET.get('end_date', END_DATE)
|
||||
|
||||
try:
|
||||
data = data_func(tscode, start_date, end_date) # 调用函数
|
||||
dict_data = data.to_dict(orient='records') if isinstance(data, pd.DataFrame) else data
|
||||
return Response(dict_data)
|
||||
except ImportError:
|
||||
return Response({'error': '模块 stock_basic 不存在'}, status=500)
|
||||
except Exception as e:
|
||||
return Response({'error': str(e)}, status=500)
|
||||
|
||||
def date_format_correction(date_str):
|
||||
"""
|
||||
日期格式矫正函数：
|
||||
1. 如果输入格式为yyyy-mm-dd则返回yyyymmdd格式
|
||||
2. 如果输入格式已经是yyyymmdd则直接返回
|
||||
3. 其他情况返回None
|
||||
|
||||
Parameters:
|
||||
date_str (str): 日期字符串
|
||||
|
||||
Returns:
|
||||
str: yyyymmdd格式的日期字符串，如果输入格式不正确则返回None
|
||||
"""
|
||||
try:
|
||||
# 先尝试解析yyyymmdd格式
|
||||
pd.to_datetime(date_str, format='%Y%m%d')
|
||||
return date_str
|
||||
except ValueError:
|
||||
try:
|
||||
# 再尝试解析yyyy-mm-dd格式
|
||||
date_obj = pd.to_datetime(date_str, format='%Y-%m-%d')
|
||||
return date_obj.strftime('%Y%m%d')
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
def get_released_report_dates(start_date: str, end_date: str) -> list:
|
||||
"""
|
||||
根据起止日期生成所有财报季末日，并去除尚未可能披露的日期。
|
||||
|
||||
:param start_date: 开始日期，格式为 "YYYYMMDD"
|
||||
:param end_date: 结束日期，格式为 "YYYYMMDD"
|
||||
:param today: 当前日期，格式为 "YYYYMMDD"，默认为系统当前日期
|
||||
:return: 已披露的财报季末日期列表，格式为 ["20240331", "20240630", ...]
|
||||
"""
|
||||
|
||||
today = datetime.today()
|
||||
|
||||
# 财报发布日期最晚规则
|
||||
report_deadlines = {
|
||||
"0331": "0430",
|
||||
"0630": "0831",
|
||||
"0930": "1031",
|
||||
"1231": "0430" # 次年4月30日
|
||||
}
|
||||
|
||||
# 生成所有财报季度末日期
|
||||
years = range(int(start_date[:4]), int(end_date[:4]) + 1)
|
||||
report_dates = [
|
||||
f"{year}{quarter}"
|
||||
for year in years
|
||||
for quarter in ["0331", "0630", "0930", "1231"]
|
||||
]
|
||||
|
||||
# 筛选日期范围内的
|
||||
report_dates = [d for d in report_dates if start_date <= d <= end_date]
|
||||
|
||||
def is_report_released(report_date_str):
|
||||
report_dt = datetime.strptime(report_date_str, "%Y%m%d")
|
||||
year = report_dt.year
|
||||
q_end = report_date_str[4:]
|
||||
|
||||
if q_end == "1231":
|
||||
deadline = datetime.strptime(f"{year+1}0430", "%Y%m%d")
|
||||
else:
|
||||
deadline = datetime.strptime(f"{year}{report_deadlines[q_end]}", "%Y%m%d")
|
||||
|
||||
return today >= deadline
|
||||
|
||||
# 返回已发布的日期
|
||||
return [d for d in report_dates if is_report_released(d)]
|
||||
|
||||
'''
|
||||
写一个函数，调用stock_basic API(说明文档：https://tushare.pro/document/2?doc_id=25) 获取个股清单
|
||||
给定交易所代码，仅查询当前仍然上市的股票
|
||||
返回：
|
||||
- ts_code: TS代码
|
||||
- symbol: 股票代码
|
||||
- name: 股票名称
|
||||
- fullname: 股票全称
|
||||
- exchange: 交易所代码 SSE上交所 SZSE深交所 BSE北交所
|
||||
- list_status: 上市状态 L上市 D退市 P暂停上市
|
||||
'''
|
||||
def get_stock_basic(exchange='SSE'):
|
||||
"""
|
||||
获取指定交易所的上市股票基本信息
|
||||
|
||||
Parameters:
|
||||
exchange (str): 交易所代码 SSE上交所 SZSE深交所 BSE北交所
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: 包含股票基本信息的DataFrame
|
||||
"""
|
||||
# 调用stock_basic接口
|
||||
df = pro.stock_basic(exchange=exchange, list_status='L',
|
||||
fields='ts_code,symbol,name,fullname,exchange,list_status')
|
||||
|
||||
return df
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试当前日期之前的财报
|
||||
print("\n测试2: 当前日期之前的财报")
|
||||
result = get_released_report_dates("20230101", "20251230")
|
||||
print(f"结果: {result}")
|
||||
@@ -0,0 +1,13 @@
|
||||
# 策略参数
|
||||
|
||||
PFAST = 10 # 快速移动平均线周期
|
||||
PSLOW = 30 # 慢速移动平均线周期
|
||||
STOP_LOSS = 0.05 # 止损比例
|
||||
TAKE_PROFIT = 0.10 # 止盈比例
|
||||
|
||||
# 初始资金
|
||||
INITIAL_CASH = 100000.0
|
||||
|
||||
# 佣金费率
|
||||
COMMISSION_BUY = 0.001 # 买入佣金费率
|
||||
COMMISSION_SELL = 0.002 # 卖出佣金费率
|
||||
@@ -0,0 +1,80 @@
|
||||
'''
|
||||
写一个方法：
|
||||
1. 接收日期范围
|
||||
2. 根据日期范围从表：xwlb_daily 查询数据：news_days, daily_sub_id, news_improve, news_title
|
||||
按news_days desc, daily_sub_id asc 排序
|
||||
3. 调用mysqlHandle.py 里的查询方法查询数据(仔细阅读video/mysqlHandle)
|
||||
'''
|
||||
# 调用mysqlHandle中的查询方法
|
||||
try:
|
||||
from .mysqlHandle import MySQLDB
|
||||
except (ImportError, SystemError):
|
||||
from mysqlHandle import MySQLDB
|
||||
import pandas as pd
|
||||
def get_xwlb(start_date, end_date):
|
||||
"""
|
||||
根据日期范围查询新闻联播数据
|
||||
|
||||
Args:
|
||||
start_date: 开始日期
|
||||
end_date: 结束日期
|
||||
|
||||
Returns:
|
||||
查询结果列表
|
||||
"""
|
||||
# SQL注入警告：使用参数化查询防止SQL注入
|
||||
sql = "xwlb_daily"
|
||||
columns = "news_days, daily_sub_id, news_improve, news_title"
|
||||
where = "news_days >= %s AND news_days <=%s order by news_days desc, daily_sub_id asc"
|
||||
params = (start_date, end_date)
|
||||
try:
|
||||
db = MySQLDB()
|
||||
result = db.query_data(sql, columns, where, params)
|
||||
print(f"查询到 {len(result)} 条记录")
|
||||
finally:
|
||||
# 关闭连接
|
||||
db.close()
|
||||
# 转换为pandas DataFrame
|
||||
|
||||
df = pd.DataFrame(result)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def get_xwlb_fine(start_date, end_date):
|
||||
"""
|
||||
根据日期范围查询新闻联播数据
|
||||
|
||||
Args:
|
||||
start_date: 开始日期
|
||||
end_date: 结束日期
|
||||
|
||||
Returns:
|
||||
查询结果列表
|
||||
"""
|
||||
# SQL注入警告：使用参数化查询防止SQL注入
|
||||
sql = "xwlb_daily_ext"
|
||||
columns = "news_date as news_days, sub_id as daily_sub_id, news_content as news_improve, news_title"
|
||||
where = "news_date >= %s AND news_date <=%s order by news_date desc, sub_id asc"
|
||||
params = (start_date, end_date)
|
||||
try:
|
||||
db = MySQLDB()
|
||||
result = db.query_data(sql, columns, where, params)
|
||||
print(f"查询到 {len(result)} 条记录")
|
||||
finally:
|
||||
# 关闭连接
|
||||
db.close()
|
||||
# 转换为pandas DataFrame
|
||||
|
||||
df = pd.DataFrame(result)
|
||||
|
||||
return df
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试代码
|
||||
start_date = "2025-01-01"
|
||||
end_date = "2025-01-31"
|
||||
result = get_xwlb(start_date, end_date)
|
||||
print("查询结果：")
|
||||
print(result.head())
|
||||
print(f"总记录数：{len(result)}")
|
||||
@@ -0,0 +1,10 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Django Home Page</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Welcome to Django Home Page!</h1>
|
||||
<p>This is the main page of our Django project.</p>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,87 @@
|
||||
from django.test import TestCase
|
||||
from .stock.stock_utils import tscodeCheck, date_format_correction, get_released_report_dates
|
||||
|
||||
|
||||
class TscodeCheckTest(TestCase):
|
||||
"""股票代码格式校验测试"""
|
||||
|
||||
def test_sz_6digit(self):
|
||||
self.assertEqual(tscodeCheck('000001'), '000001.SZ')
|
||||
|
||||
def test_sz_6digit_300(self):
|
||||
self.assertEqual(tscodeCheck('300750'), '300750.SZ')
|
||||
|
||||
def test_sh_6digit_60(self):
|
||||
self.assertEqual(tscodeCheck('600000'), '600000.SH')
|
||||
|
||||
def test_sh_6digit_68(self):
|
||||
self.assertEqual(tscodeCheck('688001'), '688001.SH')
|
||||
|
||||
def test_bj_6digit(self):
|
||||
self.assertEqual(tscodeCheck('830799'), '830799.BJ')
|
||||
|
||||
def test_9digit_pass_through(self):
|
||||
self.assertEqual(tscodeCheck('000001.SZ'), '000001.SZ')
|
||||
|
||||
def test_lowercase_to_uppercase(self):
|
||||
self.assertEqual(tscodeCheck('000001.sz'), '000001.SZ')
|
||||
|
||||
def test_invalid_length_short(self):
|
||||
with self.assertRaises(ValueError):
|
||||
tscodeCheck('12345')
|
||||
|
||||
def test_invalid_length_long(self):
|
||||
with self.assertRaises(ValueError):
|
||||
tscodeCheck('1234567890')
|
||||
|
||||
def test_invalid_suffix(self):
|
||||
with self.assertRaises(ValueError):
|
||||
tscodeCheck('000001.XX')
|
||||
|
||||
def test_not_string(self):
|
||||
with self.assertRaises(ValueError):
|
||||
tscodeCheck(123456)
|
||||
|
||||
def test_unknown_prefix(self):
|
||||
with self.assertRaises(ValueError):
|
||||
tscodeCheck('500001')
|
||||
|
||||
|
||||
class DateFormatCorrectionTest(TestCase):
|
||||
"""日期格式校正测试"""
|
||||
|
||||
def test_yyyymmdd_passthrough(self):
|
||||
self.assertEqual(date_format_correction('20240115'), '20240115')
|
||||
|
||||
def test_yyyy_mm_dd_conversion(self):
|
||||
self.assertEqual(date_format_correction('2024-01-15'), '20240115')
|
||||
|
||||
def test_invalid_format(self):
|
||||
self.assertIsNone(date_format_correction('15/01/2024'))
|
||||
|
||||
def test_empty_string(self):
|
||||
self.assertIsNone(date_format_correction(''))
|
||||
|
||||
|
||||
class GetReleasedReportDatesTest(TestCase):
|
||||
"""财报发布日期测试"""
|
||||
|
||||
def test_returns_list(self):
|
||||
result = get_released_report_dates('20230101', '20231231')
|
||||
self.assertIsInstance(result, list)
|
||||
|
||||
def test_all_dates_yyyymmdd_format(self):
|
||||
result = get_released_report_dates('20230101', '20231231')
|
||||
for d in result:
|
||||
self.assertEqual(len(d), 8)
|
||||
self.assertTrue(d.isdigit())
|
||||
|
||||
def test_start_after_end_returns_empty(self):
|
||||
result = get_released_report_dates('20251231', '20230101')
|
||||
self.assertEqual(result, [])
|
||||
|
||||
def test_single_year_quarters(self):
|
||||
"""单年内应该返回最多4个季末日期"""
|
||||
result = get_released_report_dates('20200101', '20201231')
|
||||
for d in result:
|
||||
self.assertTrue(d.endswith(('0331', '0630', '0930', '1231')))
|
||||
@@ -0,0 +1,24 @@
|
||||
from django.urls import path
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
# 其他 URL 路由
|
||||
path('python-version/', views.python_version, name='python_version'),
|
||||
path('', views.home, name='home'), # 根 URL 映射到主页视图
|
||||
path('stockbasic/', views.stockbasic, name='stockbasic'),
|
||||
path('stockparam/', views.stockparam, name='stockparam'),
|
||||
path('industrys/', views.industrys, name='industrys'),
|
||||
path('stockinfo/', views.stockInfo, name='stockInfo'),
|
||||
path('stockep/', views.stockep, name='stockep'),
|
||||
path('quarterlyEps/', views.quarterlyEps, name='quarterlyEps'),
|
||||
path('indexByName/', views.indexByName, name='indexByName'),
|
||||
path('indexDatas/', views.indexDatas, name='indexDatas'),
|
||||
path('dailymargin/', views.dailyMargin, name='dailyMargin'),
|
||||
path('stockmargin/', views.stockMargin, name='stockMargin'),
|
||||
path('stockep/', views.stockep, name='stockep'),
|
||||
path('finance/', views.getFinaData, name='getFinaData'),
|
||||
path('getdiv/', views.getDivData, name='getDivData'),
|
||||
path('getdivak/', views.getDivDataAkshare, name='getDivDataAkshare'),
|
||||
path('xwlbNews/', views.xwlbNews, name='xwlbNews'),
|
||||
path('xwlbFine/', views.xwlbFine, name='xwlbFine'),
|
||||
]
|
||||
@@ -0,0 +1,94 @@
|
||||
import os
|
||||
import mysql.connector
|
||||
from mysql.connector import Error
|
||||
|
||||
|
||||
class MySQLDB:
|
||||
def __init__(self, host=None, port=None, username=None, password=None, database=None):
|
||||
self.host = host or os.getenv('MYSQL_HOST', 'localhost')
|
||||
self.port = port or int(os.getenv('MYSQL_PORT', '3306'))
|
||||
self.username = username or os.getenv('MYSQL_USER', 'myquant')
|
||||
self.password = password or os.getenv('MYSQL_PASSWORD', '')
|
||||
self.database = database or os.getenv('MYSQL_DATABASE', 'myquant')
|
||||
self.connection = None
|
||||
self.connect()
|
||||
|
||||
def connect(self):
|
||||
"""连接数据库"""
|
||||
try:
|
||||
self.connection = mysql.connector.connect(
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
user=self.username,
|
||||
password=self.password,
|
||||
database=self.database
|
||||
)
|
||||
if self.connection.is_connected():
|
||||
print("成功连接到MySQL数据库")
|
||||
except Error as e:
|
||||
print(f"连接错误: {e}")
|
||||
|
||||
def insert_data(self, table, data):
|
||||
"""插入数据"""
|
||||
try:
|
||||
cursor = self.connection.cursor()
|
||||
columns = ', '.join(data.keys())
|
||||
placeholders = ', '.join(['%s'] * len(data))
|
||||
query = f"INSERT INTO {table} ({columns}) VALUES ({placeholders})"
|
||||
|
||||
cursor.execute(query, tuple(data.values()))
|
||||
self.connection.commit()
|
||||
print(f"成功插入数据，影响行数: {cursor.rowcount}")
|
||||
return cursor.lastrowid
|
||||
except Error as e:
|
||||
print(f"插入错误: {e}")
|
||||
return None
|
||||
finally:
|
||||
if cursor:
|
||||
cursor.close()
|
||||
|
||||
def query_data(self, table, columns="*", where=None, params=None):
|
||||
"""查询数据"""
|
||||
try:
|
||||
cursor = self.connection.cursor(dictionary=True)
|
||||
|
||||
query = f"SELECT {columns} FROM {table}"
|
||||
if where:
|
||||
query += f" WHERE {where}"
|
||||
print(query)
|
||||
cursor.execute(query, params or ())
|
||||
result = cursor.fetchall()
|
||||
return result
|
||||
except Error as e:
|
||||
print(f"查询错误: {e}")
|
||||
return []
|
||||
finally:
|
||||
if cursor:
|
||||
cursor.close()
|
||||
|
||||
def update_data(self, table, data, where, params=None):
|
||||
"""更新数据"""
|
||||
try:
|
||||
cursor = self.connection.cursor()
|
||||
|
||||
set_clause = ', '.join([f"{key} = %s" for key in data.keys()])
|
||||
query = f"UPDATE {table} SET {set_clause} WHERE {where}"
|
||||
|
||||
all_params = tuple(data.values()) + (params if params else ())
|
||||
|
||||
cursor.execute(query, all_params)
|
||||
self.connection.commit()
|
||||
print(f"成功更新数据，影响行数: {cursor.rowcount}")
|
||||
return cursor.rowcount
|
||||
except Error as e:
|
||||
print(f"更新错误: {e}")
|
||||
return 0
|
||||
finally:
|
||||
if cursor:
|
||||
cursor.close()
|
||||
|
||||
def close(self):
|
||||
"""关闭数据库连接"""
|
||||
if self.connection and self.connection.is_connected():
|
||||
self.connection.close()
|
||||
print("数据库连接已关闭")
|
||||
@@ -0,0 +1,436 @@
|
||||
import env # 加载 .env 到环境变量
|
||||
import os
|
||||
import dashscope
|
||||
import pydub
|
||||
from pydub import AudioSegment
|
||||
from pydub.silence import split_on_silence
|
||||
from dashscope.audio.asr import Recognition
|
||||
from dashscope import Generation
|
||||
from http import HTTPStatus
|
||||
from mysqlHandle import MySQLDB
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
# 设置环境变量
|
||||
#os.environ["DASHSCOPE_API_KEY"] = "sk-d2d65b726068445b98b88fc3b675dbf1" # 替换为你的API Key
|
||||
|
||||
def convert_mp3_to_wav(mp3_path, output_wav_path):
|
||||
"""
|
||||
将MP3文件转换为16kHz单声道WAV格式，这是Qwen3-ASR-Flash模型的推荐格式
|
||||
参数:
|
||||
mp3_path (str): MP3文件路径
|
||||
output_wav_path (str): 输出WAV文件路径
|
||||
返回值:
|
||||
str: 转换后的WAV文件路径
|
||||
"""
|
||||
logger.info(f"开始转换MP3到WAV: {mp3_path}")
|
||||
# 加载MP3文件
|
||||
audio = AudioSegment.from_file(mp3_path, format="mp3")
|
||||
# 转换为16kHz采样率、单声道、16位深度
|
||||
audio = audio.set_frame_rate(16000).set_channels(1)
|
||||
# 导出为WAV格式
|
||||
audio.export(output_wav_path, format="wav")
|
||||
logger.info(f"✓ MP3转换完成: {output_wav_path}")
|
||||
#print(f"✓ MP3转换完成: {output_wav_path}")
|
||||
return output_wav_path
|
||||
|
||||
def split_audio_by_fixed_duration(audio_path, chunk_duration, output_folder):
|
||||
"""
|
||||
将音频文件按固定时长分割成多个片段
|
||||
参数:
|
||||
audio_path (str): 音频文件路径
|
||||
chunk_duration (int): 分片时长(毫秒)
|
||||
output_folder (str): 输出文件夹路径
|
||||
返回值:
|
||||
list: 分片文件路径列表
|
||||
"""
|
||||
# 加载音频文件
|
||||
audio = AudioSegment.from_file(audio_path)
|
||||
# 计算总时长(毫秒)
|
||||
total_duration = len(audio)
|
||||
# 分片数
|
||||
num_chunks = total_duration // chunk_duration + 1
|
||||
# 存储分片文件路径
|
||||
chunks = []
|
||||
|
||||
# 创建输出文件夹
|
||||
os.makedirs(output_folder, exist_ok=True)
|
||||
|
||||
logger.info(f"开始音频分割，总时长: {total_duration/1000:.1f}秒，将分割为{num_chunks}个片段")
|
||||
|
||||
for i in range(num_chunks):
|
||||
# 计算当前分片的起始和结束时间
|
||||
start_time = i * chunk_duration
|
||||
end_time = (i + 1) * chunk_duration
|
||||
# 提取分片音频
|
||||
chunk = audio[start_time:end_time]
|
||||
# 生成文件名
|
||||
chunk_name = f"chunk_{i}.wav"
|
||||
chunk_path = os.path.join(output_folder, chunk_name)
|
||||
# 导出分片音频
|
||||
chunk.export(chunk_path, format="wav")
|
||||
chunks.append(chunk_path)
|
||||
|
||||
# 打印处理进度
|
||||
progress = (i + 1) / num_chunks * 100
|
||||
logger.info(f"✓ 已完成分片 {i+1}/{num_chunks} ({progress:.1f}%)")
|
||||
|
||||
logger.info(f"✓ 音频分割完成，共生成{len(chunks)}个分片文件")
|
||||
return chunks
|
||||
|
||||
def split_audio_by_smart_silence(audio_path, min_silence_len, silence_thresh, output_folder):
|
||||
"""
|
||||
将音频文件按智能静音检测方式分割成多个片段，每段不超过3分钟
|
||||
参数:
|
||||
audio_path (str): 音频文件路径
|
||||
min_silence_len (int): 最小静音长度(毫秒)
|
||||
silence_thresh (int): 静音阈值(dBFS)
|
||||
output_folder (str): 输出文件夹路径
|
||||
返回值:
|
||||
list: 分片文件路径列表
|
||||
"""
|
||||
# 加载音频文件
|
||||
audio = AudioSegment.from_file(audio_path, format="wav")
|
||||
# 按静音分割
|
||||
segments = split_on_silence(
|
||||
audio,
|
||||
# 静音超过700毫秒则分割
|
||||
min_silence_len=min_silence_len,
|
||||
# 静音阈值为-40dBFS
|
||||
silence_thresh=silence_thresh,
|
||||
# 保留静音部分
|
||||
keep_silence=400
|
||||
)
|
||||
|
||||
logger.info(f"✓ 静音分割完成，共{len(segments)}个初始片段")
|
||||
|
||||
# 合并过短的片段
|
||||
merged_segments = []
|
||||
current_segment = None
|
||||
for segment in segments:
|
||||
if current_segment is None:
|
||||
current_segment = segment
|
||||
else:
|
||||
# 合并当前片段和新片段
|
||||
temp_segment = current_segment + segment
|
||||
# 如果合并后的片段超过3分钟，则单独保存当前片段
|
||||
if len(temp_segment) > 180000: # 3分钟=180,000毫秒
|
||||
merged_segments.append(current_segment)
|
||||
current_segment = segment
|
||||
else:
|
||||
current_segment = temp_segment
|
||||
# 添加最后一个片段
|
||||
if current_segment is not None:
|
||||
merged_segments.append(current_segment)
|
||||
|
||||
logger.info(f"✓ 片段合并完成，共{len(merged_segments)}个最终片段")
|
||||
|
||||
# 存储分片文件路径
|
||||
chunks = []
|
||||
|
||||
# 创建输出文件夹
|
||||
os.makedirs(output_folder, exist_ok=True)
|
||||
|
||||
logger.info(f"开始导出音频片段到: {output_folder}")
|
||||
|
||||
for i, segment in enumerate(merged_segments):
|
||||
# 生成文件名
|
||||
chunk_name = f"chunk_{i}.wav"
|
||||
chunk_path = os.path.join(output_folder, chunk_name)
|
||||
# 导出分片音频
|
||||
segment.export(chunk_path, format="wav")
|
||||
chunks.append(chunk_path)
|
||||
|
||||
# 打印处理进度
|
||||
progress = (i + 1) / len(merged_segments) * 100
|
||||
logger.info(f"✓ 已完成分片 {i+1}/{len(merged_segments)} ({progress:.1f}%)")
|
||||
|
||||
logger.info(f"✓ 智能静音分割完成，共生成{len(chunks)}个分片文件")
|
||||
return chunks
|
||||
|
||||
|
||||
def transcribe_audio(audio_path):
|
||||
"""
|
||||
使用Paraformer实时语音识别模型（通过本地文件）转录音频文件
|
||||
参数:
|
||||
audio_path (str): 音频文件路径（必须是16kHz单声道WAV）
|
||||
返回值:
|
||||
str: 识别文本，如果失败返回空字符串
|
||||
"""
|
||||
try:
|
||||
# 确保音频文件存在
|
||||
if not os.path.exists(audio_path):
|
||||
logger.error(f"音频文件不存在: {audio_path}")
|
||||
return ""
|
||||
dashscope.api_key = os.getenv('DASHSCOPE_API_KEY', '')
|
||||
# 创建识别对象
|
||||
recognition = Recognition(
|
||||
model='paraformer-realtime-v2', # 使用实时识别模型
|
||||
format='wav',
|
||||
sample_rate=16000,
|
||||
language_hints=['zh','en'], # 中文和英文
|
||||
callback=None
|
||||
)
|
||||
|
||||
# 调用识别
|
||||
logger.info(f"开始识别音频: {audio_path}")
|
||||
result = recognition.call(audio_path)
|
||||
text=[]
|
||||
if result.status_code == HTTPStatus.OK:
|
||||
# 提取识别结果
|
||||
logger.info(f"✓ {audio_path} 识别成功")
|
||||
sentence = result.get_sentence()
|
||||
text.append(merge_transcripts(sentence))
|
||||
logger.info(f"识别文本长度: {len(text[0])}")
|
||||
text.append(analyze_and_correct_text(text[0]))
|
||||
return text
|
||||
else:
|
||||
logger.error(f"❌ 任务失败: {result.message}")
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.error(f"识别过程中发生异常: {e}")
|
||||
return ""
|
||||
|
||||
def merge_transcripts(transcripts):
|
||||
"""
|
||||
将多段识别文本合并成完整句子（保留原始段落逻辑，用空格连接）
|
||||
参数:
|
||||
transcripts (list): 识别结果列表，每个元素为字典{'text': '识别文本'}
|
||||
返回:
|
||||
str: 合并后的完整文本
|
||||
"""
|
||||
# 输入参数检查
|
||||
if not transcripts:
|
||||
return ""
|
||||
|
||||
# 确保transcripts是可迭代对象
|
||||
if not hasattr(transcripts, '__iter__'):
|
||||
return ""
|
||||
|
||||
try:
|
||||
# 提取所有有效的text字段
|
||||
texts = []
|
||||
for t in transcripts:
|
||||
try:
|
||||
# 检查是否为字典类型且包含text字段
|
||||
if isinstance(t, dict) and 'text' in t and t['text']:
|
||||
text = t['text']
|
||||
# 确保text是字符串类型
|
||||
if isinstance(text, str) and text.strip():
|
||||
texts.append(text.strip())
|
||||
except (KeyError, TypeError, AttributeError):
|
||||
# 忽略单个元素的处理错误，继续处理其他元素
|
||||
continue
|
||||
|
||||
# 用空格连接所有段落（根据实际需求可调整连接符）
|
||||
return " ".join(texts) if texts else ""
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"合并转录文本时发生错误: {e}")
|
||||
return ""
|
||||
def text_correction(text):
|
||||
"""
|
||||
使用通义千问模型修正文本中的错误和标点符号
|
||||
参数:
|
||||
text (str): 需要修正的文本
|
||||
返回值:
|
||||
str: 修正后的文本
|
||||
"""
|
||||
logger.info("开始文本修正...")
|
||||
|
||||
# 构建修正提示词
|
||||
correction_prompt = """请仔细检查以下文本，修正其中的错误：
|
||||
1. 错别字和语法错误
|
||||
2. 标点符号使用错误
|
||||
3. 语句不通顺的地方
|
||||
4. 逻辑不清晰的部分
|
||||
|
||||
请直接返回修正后的完整文本，不要添加任何解释说明。"""
|
||||
|
||||
# 构建消息列表
|
||||
messages = [
|
||||
{"role": "system", "content": "你是一个专业的文本校对助手，擅长修正文本中的各种错误。"},
|
||||
{"role": "user", "content": correction_prompt},
|
||||
{"role": "user", "content": text}
|
||||
]
|
||||
|
||||
logger.info("调用通义千问模型进行文本修正...")
|
||||
# 调用DashScope文本生成接口
|
||||
response = Generation.call(
|
||||
model="qwen-plus",
|
||||
messages=messages,
|
||||
max_tokens=30000,
|
||||
temperature=0.1, # 使用较低的温度以提高确定性
|
||||
top_p=0.5
|
||||
)
|
||||
|
||||
# 检查API调用是否成功
|
||||
if response.status_code != 200:
|
||||
logger.warning(f"❌ 文本修正API调用失败: {response.message}")
|
||||
raise Exception(f"文本修正API调用失败: {response.message}")
|
||||
|
||||
logger.info("✓ 文本修正完成")
|
||||
# 返回修正后的文本
|
||||
return response.output.text
|
||||
|
||||
def analyze_and_correct_text(text):
|
||||
"""
|
||||
分析文本并自动修正错误
|
||||
参数:
|
||||
text (str): 待分析和修正的文本
|
||||
prompt (str): 分析提示词
|
||||
返回值:
|
||||
tuple: (修正后的文本, 分析结果)
|
||||
"""
|
||||
logger.info("开始文本分析和修正流程...")
|
||||
|
||||
# 首先修正文本错误
|
||||
corrected_text = text_correction(text)
|
||||
logger.info(f"原始文本长度: {len(text)}")
|
||||
logger.info(f"修正后文本长度: {len(corrected_text)}")
|
||||
|
||||
# 使用修正后的文本进行分析
|
||||
# analysis_result = analyze_text(corrected_text, prompt)
|
||||
|
||||
return corrected_text
|
||||
def analyze_text(text, prompt):
|
||||
"""
|
||||
使用通义千问模型分析文本
|
||||
参数:
|
||||
text (str): 待分析文本
|
||||
prompt (str): 分析提示词
|
||||
返回值:
|
||||
str: 分析结果
|
||||
"""
|
||||
logger.info("开始文本分析...")
|
||||
# 设置系统提示
|
||||
system_prompt = "你是一个专业的文本分析助手，擅长根据提示词对长文本进行深入分析。"
|
||||
# 构建消息列表
|
||||
messages = [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": prompt},
|
||||
{"role": "user", "content": text}
|
||||
]
|
||||
|
||||
logger.info("调用通义千问模型进行文本分析...")
|
||||
# 调用DashScope文本生成接口
|
||||
response = Generation.call(
|
||||
model="qwen-plus", # 使用通义千问Plus模型进行分析
|
||||
messages=messages,
|
||||
max_tokens=8190, # 控制生成文本的最大长度
|
||||
temperature=0.3, # 控制生成文本的确定性
|
||||
top_p=0.7 # 控制生成文本的多样性
|
||||
)
|
||||
|
||||
# 检查API调用是否成功
|
||||
if response.status_code != 200:
|
||||
logger.error(f"❌ API调用失败: {response.message}")
|
||||
raise Exception(f"API调用失败: {response.message}")
|
||||
|
||||
logger.info("✓ 文本分析完成")
|
||||
# 返回分析结果
|
||||
return response.output.text
|
||||
|
||||
def process_long_audio(mp3_path, output_folder, date_str):
|
||||
"""
|
||||
处理长音频文件，分割、识别并分析
|
||||
参数:
|
||||
mp3_path (str): MP3文件路径
|
||||
prompt (str): 分析提示词
|
||||
output_folder (str): 输出文件夹路径
|
||||
返回值:
|
||||
str: 分析结果
|
||||
"""
|
||||
logger.info("开始处理长音频...")
|
||||
|
||||
# 转换MP3为WAV格式
|
||||
logger.info("步骤1/4: 转换MP3为WAV格式")
|
||||
wav_path = convert_mp3_to_wav(
|
||||
mp3_path, os.path.join(output_folder, "input.wav")
|
||||
)
|
||||
|
||||
# 分割音频
|
||||
# 可以选择固定分片或智能静音分割
|
||||
# chunks = split_audio_by_fixed_duration(wav_path, 180000, output_folder)
|
||||
logger.info("步骤2/4: 智能静音分割音频")
|
||||
chunks = split_audio_by_smart_silence(
|
||||
wav_path, 700, -40, output_folder
|
||||
)
|
||||
|
||||
# 存储所有识别文本
|
||||
transcribed_text = ""
|
||||
|
||||
# 识别每个分片
|
||||
logger.info(f"步骤3/4: 开始识别音频分片，共{len(chunks)}个分片")
|
||||
for i, chunk_path in enumerate(chunks):
|
||||
try:
|
||||
logger.info(f"识别进度: {i+1}/{len(chunks)} ({((i+1)/len(chunks)*100):.1f}%)")
|
||||
# 调用音频识别API
|
||||
text = transcribe_audio(chunk_path)
|
||||
|
||||
"""
|
||||
if not text[1].startswith('今天的新闻联播节目播送完毕'):
|
||||
prompt='请分析所给文本的新闻内容，返回一个简短标题'
|
||||
text.append(analyze_text(text[1],prompt))
|
||||
else:
|
||||
text.append('')
|
||||
"""
|
||||
# 新闻标题留空
|
||||
text.append('')
|
||||
# 添加到总文本
|
||||
#transcribed_text += text + "\n"
|
||||
# 删除临时文件
|
||||
os.remove(chunk_path)
|
||||
# 初始化数据库连接
|
||||
db = MySQLDB() # 使用默认参数连接数据库
|
||||
try:
|
||||
# 插入数据示例
|
||||
user_data = {
|
||||
"news_days": date_str,
|
||||
"daily_sub_id": i,
|
||||
"news_raw": text[0],
|
||||
"news_improve": text[1],
|
||||
"news_title": text[2]
|
||||
}
|
||||
user_id = db.insert_data("xwlb_daily", user_data)
|
||||
finally:
|
||||
# 关闭连接
|
||||
db.close()
|
||||
except Exception as e:
|
||||
logger.error(f"识别失败: {chunk_path}, 错误: {e}")
|
||||
# 可以在这里添加重试逻辑
|
||||
|
||||
# 分析识别文本
|
||||
"""
|
||||
print("步骤4/4: 分析识别文本")
|
||||
print(f"识别文本长度: {len(transcribed_text)}")
|
||||
print(f"识别文本内容: {transcribed_text}")
|
||||
if len(transcribed_text) < 1:
|
||||
print("识别文本为空，跳过分析处理")
|
||||
return "识别文本为空，无法进行分析"
|
||||
analysis_result = analyze_text(transcribed_text, prompt)
|
||||
"""
|
||||
logger.info("✓ 长音频处理完成")
|
||||
# 返回分析结果
|
||||
return ""
|
||||
|
||||
# 使用示例
|
||||
if __name__ == "__main__":
|
||||
# MP3文件路径
|
||||
mp3_path = "20251002.mp3"
|
||||
# 分析提示词
|
||||
prompt = "请总结这段由中国中央电视台新闻联播音频转为文字的文本，理解其主要内容并提取其中的关键信息。"
|
||||
# 输出文件夹
|
||||
output_folder = "audio_processing"
|
||||
|
||||
# 处理长音频
|
||||
try:
|
||||
result = process_long_audio(
|
||||
mp3_path, prompt, output_folder
|
||||
)
|
||||
# 打印分析结果
|
||||
print("分析结果:\n")
|
||||
print(result)
|
||||
except Exception as e:
|
||||
print(f"处理失败: {e}")
|
||||
@@ -0,0 +1,267 @@
|
||||
import env # 加载 .env 到环境变量
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
from typing import Optional, Dict, Any
|
||||
import os
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class DeepSeekAPI:
|
||||
def __init__(self, api_key: Optional[str] = None):
|
||||
"""
|
||||
初始化DeepSeek API客户端
|
||||
|
||||
Args:
|
||||
api_key: DeepSeek API密钥，如果为None则从环境变量获取
|
||||
"""
|
||||
self.api_key = api_key or os.getenv('DEEPSEEK_API_KEY')
|
||||
if not self.api_key:
|
||||
logger.warning("API密钥未提供且环境变量DEEPSEEK_API_KEY未设置")
|
||||
|
||||
self.api_url = "https://api.deepseek.com/v1/chat/completions"
|
||||
self.max_retries = 3
|
||||
self.retry_delay = 2 # 秒
|
||||
|
||||
# 默认系统提示词
|
||||
self.default_system_prompt = """你是一个专业的AI助手，能够准确理解用户需求并提供高质量的回答。
|
||||
请根据用户的输入进行适当的处理和分析，保持回答的专业性和准确性。注意：所处理文字来自中央电视台新闻联播节目转文字，请在内容审查时重点考虑。"""
|
||||
|
||||
def _handle_api_error(self, response: requests.Response) -> str:
|
||||
"""
|
||||
处理API错误响应
|
||||
|
||||
Args:
|
||||
response: API响应对象
|
||||
|
||||
Returns:
|
||||
错误描述信息
|
||||
"""
|
||||
error_msg = f"API请求失败: {response.status_code} {response.reason}"
|
||||
|
||||
try:
|
||||
error_data = response.json()
|
||||
if 'error' in error_data:
|
||||
error_msg += f" - {error_data['error'].get('message', '未知错误')}"
|
||||
logger.error(f"API错误详情: {error_data}")
|
||||
except json.JSONDecodeError:
|
||||
error_msg += f" - 响应内容: {response.text[:200]}"
|
||||
|
||||
return error_msg
|
||||
|
||||
def _make_api_request(self, payload: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
发送API请求并处理响应
|
||||
|
||||
Args:
|
||||
payload: 请求数据
|
||||
|
||||
Returns:
|
||||
API响应数据
|
||||
|
||||
Raises:
|
||||
Exception: 当所有重试都失败时抛出异常
|
||||
"""
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {self.api_key}"
|
||||
}
|
||||
|
||||
last_exception = None
|
||||
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
logger.info(f"发送API请求 (尝试 {attempt + 1}/{self.max_retries})")
|
||||
|
||||
response = requests.post(
|
||||
self.api_url,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=60 # 60秒超时
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
elif response.status_code == 400:
|
||||
# 400错误通常是请求格式问题，不需要重试
|
||||
error_msg = self._handle_api_error(response)
|
||||
raise Exception(f"请求参数错误: {error_msg}")
|
||||
elif response.status_code == 401:
|
||||
# 401未授权错误，不需要重试
|
||||
raise Exception("API密钥无效或未授权，请检查您的API密钥")
|
||||
elif response.status_code == 429:
|
||||
# 速率限制，需要重试
|
||||
logger.warning("达到速率限制，等待后重试...")
|
||||
time.sleep(self.retry_delay * (attempt + 1))
|
||||
continue
|
||||
elif 500 <= response.status_code < 600:
|
||||
# 服务器错误，需要重试
|
||||
logger.warning(f"服务器错误 {response.status_code}，等待后重试...")
|
||||
time.sleep(self.retry_delay * (attempt + 1))
|
||||
continue
|
||||
else:
|
||||
error_msg = self._handle_api_error(response)
|
||||
raise Exception(f"API请求失败: {error_msg}")
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
last_exception = Exception(f"请求超时 (尝试 {attempt + 1})")
|
||||
logger.warning(f"请求超时，等待后重试...")
|
||||
time.sleep(self.retry_delay * (attempt + 1))
|
||||
|
||||
except requests.exceptions.ConnectionError:
|
||||
last_exception = Exception(f"网络连接错误 (尝试 {attempt + 1})")
|
||||
logger.warning(f"网络连接错误，等待后重试...")
|
||||
time.sleep(self.retry_delay * (attempt + 1))
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
last_exception = Exception(f"请求异常: {str(e)}")
|
||||
logger.warning(f"请求异常，等待后重试...")
|
||||
time.sleep(self.retry_delay * (attempt + 1))
|
||||
|
||||
# 所有重试都失败
|
||||
if last_exception:
|
||||
raise last_exception
|
||||
else:
|
||||
raise Exception("API请求失败，未知错误")
|
||||
|
||||
def process_text(self,
|
||||
prompt: str,
|
||||
text: str,
|
||||
system_prompt: Optional[str] = None,
|
||||
model: str = "deepseek-chat",
|
||||
temperature: float = 0.7,
|
||||
max_tokens: int = 2000) -> str:
|
||||
"""
|
||||
处理文本的通用方法
|
||||
|
||||
Args:
|
||||
prompt: 用户提示词
|
||||
text: 需要处理的文本（约1万字符）
|
||||
system_prompt: 系统提示词，如果为None则使用默认值
|
||||
model: 使用的模型
|
||||
temperature: 生成温度
|
||||
max_tokens: 最大生成token数
|
||||
|
||||
Returns:
|
||||
处理后的文本
|
||||
|
||||
Raises:
|
||||
Exception: 当处理失败时抛出包含详细信息的异常
|
||||
"""
|
||||
# 输入验证
|
||||
if not self.api_key:
|
||||
raise Exception("API密钥未设置，请提供api_key或设置DEEPSEEK_API_KEY环境变量")
|
||||
|
||||
if not prompt or not text:
|
||||
raise Exception("prompt和text不能为空")
|
||||
|
||||
# 检查文本长度（约1万字符）
|
||||
if len(text) > 15000: # 留一些余量
|
||||
logger.warning(f"输入文本长度({len(text)}字符)较长，可能会超过上下文限制")
|
||||
|
||||
# 准备系统提示词
|
||||
system_content = system_prompt or self.default_system_prompt
|
||||
|
||||
# 构建消息
|
||||
messages = [
|
||||
{"role": "system", "content": system_content},
|
||||
{"role": "user", "content": f"{prompt}\n\n文本内容：\n{text}"}
|
||||
]
|
||||
|
||||
# 构建请求数据
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
"stream": False
|
||||
}
|
||||
|
||||
try:
|
||||
# 发送API请求
|
||||
response_data = self._make_api_request(payload)
|
||||
|
||||
# 解析响应
|
||||
if 'choices' in response_data and len(response_data['choices']) > 0:
|
||||
result = response_data['choices'][0]['message']['content']
|
||||
logger.info("文本处理成功完成")
|
||||
return result.strip()
|
||||
else:
|
||||
raise Exception("API响应格式异常，未找到有效结果")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"文本处理失败: {str(e)}")
|
||||
raise Exception(f"文本处理失败: {str(e)}")
|
||||
|
||||
def process_text_with_fallback(self,
|
||||
prompt: str,
|
||||
text: str,
|
||||
system_prompt: Optional[str] = None,
|
||||
**kwargs) -> str:
|
||||
"""
|
||||
带降级处理的文本处理方法
|
||||
|
||||
Args:
|
||||
prompt: 用户提示词
|
||||
text: 需要处理的文本
|
||||
system_prompt: 系统提示词
|
||||
**kwargs: 其他参数
|
||||
|
||||
Returns:
|
||||
处理后的文本，如果API调用失败则返回降级结果
|
||||
"""
|
||||
try:
|
||||
return self.process_text(prompt, text, system_prompt, **kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"API调用失败，使用降级处理: {str(e)}")
|
||||
# 这里可以添加降级逻辑，比如返回原始文本或简单处理
|
||||
return f"处理失败，返回原始文本（错误: {str(e)}）\n\n{text}"
|
||||
|
||||
# 使用示例
|
||||
def deepseek_text(text, prompt):
|
||||
# 初始化API客户端
|
||||
# 方式1: 直接传入API密钥
|
||||
# api_client = DeepSeekAPI(api_key="your_deepseek_api_key_here")
|
||||
|
||||
# 方式2: 从环境变量读取（推荐）
|
||||
api_client = DeepSeekAPI() # API key 从环境变量 DEEPSEEK_API_KEY 读取
|
||||
|
||||
# 示例文本（约1万字符）
|
||||
# sample_text = "这里是你的长文本内容..." * 500 # 模拟长文本
|
||||
|
||||
# 自定义系统提示词（可选）
|
||||
custom_system_prompt = "你是一个专业的文本分析助手，擅长根据提示词对长文本进行深入分析。"
|
||||
|
||||
try:
|
||||
# 处理文本
|
||||
result = api_client.process_text(
|
||||
model="deepseek-reasoner",
|
||||
prompt=prompt,
|
||||
text=text,
|
||||
system_prompt=custom_system_prompt,
|
||||
temperature=0.5,
|
||||
max_tokens=20000
|
||||
)
|
||||
|
||||
#print("处理结果:")
|
||||
#print(result)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"处理失败: {e}")
|
||||
|
||||
# 使用降级方法
|
||||
fallback_result = api_client.process_text_with_fallback(
|
||||
prompt=prompt,
|
||||
text=text,
|
||||
system_prompt=custom_system_prompt
|
||||
)
|
||||
#print("降级处理结果:")
|
||||
#print(fallback_result)
|
||||
return result
|
||||
|
||||
if __name__ == "__main__":
|
||||
deepseek_text()
|
||||
@@ -0,0 +1,27 @@
|
||||
"""video 模块独立 .env 加载器 — 与 djapi/env_loader.py 功能一致但非 Django 依赖"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _load_dotenv():
|
||||
"""从项目根目录 .env 加载环境变量（不覆盖已有）"""
|
||||
# video/env.py → video/ → api/ → djapi/ (项目根)
|
||||
base_dir = Path(__file__).resolve().parent.parent.parent
|
||||
dotenv_path = base_dir / '.env'
|
||||
|
||||
if not dotenv_path.exists():
|
||||
return
|
||||
|
||||
with open(dotenv_path) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#') or '=' not in line:
|
||||
continue
|
||||
key, _, value = line.partition('=')
|
||||
key = key.strip()
|
||||
value = value.strip().strip('"').strip("'")
|
||||
if key and key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
_load_dotenv()
|
||||
@@ -0,0 +1,231 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import re,os,subprocess
|
||||
from datetime import timedelta, date
|
||||
import yt_dlp
|
||||
from audioRead import *
|
||||
from newsProcess import news_to_db
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def get_xwlb_video_link(url):
|
||||
"""
|
||||
从央视网新闻联播页面抓取历史完整版视频链接
|
||||
"""
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
'Referer': 'https://tv.cctv.com/'
|
||||
}
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=headers, timeout=10)
|
||||
response.encoding = 'utf-8'
|
||||
if response.status_code != 200:
|
||||
logger.error(f"请求失败，状态码: {response.status_code}")
|
||||
#print(f"请求失败，状态码: {response.status_code}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"请求异常: {e}")
|
||||
return []
|
||||
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
video_links = []
|
||||
|
||||
# 查找所有包含“完整版《新闻联播》”的链接
|
||||
# 方法1: 查找包含 <i class="sql0">完整版</i>《新闻联播》 的 a 标签
|
||||
for a_tag in soup.find_all('a', href=True):
|
||||
# 检查文本中是否包含“完整版”和“新闻联播”
|
||||
title_text = a_tag.get_text(strip=True)
|
||||
inner_html = str(a_tag)
|
||||
|
||||
# 判断是否是“完整版《新闻联播》”的链接
|
||||
if ('完整版' in title_text and '新闻联播' in title_text) or \
|
||||
(re.search(r'<i[^>]*>完整版</i>\s*《新闻联播》', inner_html)):
|
||||
|
||||
video_url = a_tag['href']
|
||||
# 提取日期信息（从标题或链接中）
|
||||
date_match = re.search(r'\d{8}', title_text)
|
||||
if not date_match:
|
||||
# 从链接中提取日期，如 /2025/09/25/...VID...250925.shtml
|
||||
date_match = re.search(r'/(\d{4})/(\d{2})/(\d{2})/', video_url)
|
||||
if date_match:
|
||||
year, month, day = date_match.groups()
|
||||
date_str = f"{year}{month}{day}"
|
||||
else:
|
||||
date_str = "未知日期"
|
||||
else:
|
||||
date_str = date_match.group()
|
||||
|
||||
video_links.append({
|
||||
'date': date_str,
|
||||
'title': title_text.strip(),
|
||||
'url': video_url,
|
||||
'page_url': url
|
||||
})
|
||||
logger.info(f"✅ 找到新闻联播完整版: {date_str} -> {video_url}")
|
||||
|
||||
return video_url
|
||||
|
||||
def xwlb_urls(start: str, end: str):
|
||||
"""
|
||||
start/end 格式 '20240925'
|
||||
返回列表，如 ['https://tv.cctv.com/lm/xwlb/day/20240925.shtml', ...]
|
||||
"""
|
||||
d0 = date(int(start[:4]), int(start[4:6]), int(start[6:8]))
|
||||
|
||||
d1 = date(int(end[:4]), int(end[4:6]), int(end[6:8]))
|
||||
urls = []
|
||||
for n in range((d1 - d0).days + 1):
|
||||
day = d0 + timedelta(days=n)
|
||||
urls.append({"url": f"https://tv.cctv.com/lm/xwlb/day/{day:%Y%m%d}.shtml", "date": f"{day:%Y%m%d}"})
|
||||
#print(urls)
|
||||
return urls
|
||||
|
||||
def get_all_video_links(start: str, end: str):
|
||||
base_urls=xwlb_urls(start,end)
|
||||
#print(base_urls)
|
||||
video_urls = []
|
||||
for url in base_urls:
|
||||
video=get_xwlb_video_link(url['url'])
|
||||
video_urls.append({"url":video,"date":url['date']})
|
||||
|
||||
return video_urls
|
||||
|
||||
'''
|
||||
get_xwlb_video_link() 方法获得的url，
|
||||
urls like: https://tv.cctv.com/2024/10/30/VIDEUlPz1Qusy41JFQj3LMLd241030.shtml
|
||||
通过yt-dlp下载视频，保存为mp4文件，并用ffmpeg提取音频为mp3文件,文件名使用url 的日期部分，如上面的url应保存为 20241030.mp4 和 20241030.mp3
|
||||
文件保存路径为当前目录下的 xwlb_video 文件夹，若不存在则创建。
|
||||
'''
|
||||
|
||||
|
||||
def download_and_extract_audio(video_url,date_str,download_dir):
|
||||
"""
|
||||
使用yt-dlp下载视频并提取音频
|
||||
"""
|
||||
# 从URL中提取日期
|
||||
|
||||
os.makedirs(download_dir, exist_ok=True)
|
||||
|
||||
# 构建文件路径
|
||||
mp4_path = os.path.join(download_dir, f"{date_str}.mp4")
|
||||
mp3_path = os.path.join(download_dir, f"{date_str}.mp3")
|
||||
|
||||
try:
|
||||
# 使用yt-dlp库下载视频
|
||||
logger.info(f"📥 开始下载 {date_str} 的视频...")
|
||||
# 配置yt-dlp选项
|
||||
ydl_opts = {
|
||||
'outtmpl': mp4_path,
|
||||
'format': 'best[ext=mp4]/best',
|
||||
'progress_hooks': [lambda d: print(f"\r📥 下载进度: {d.get('_percent_str', 'N/A').strip()} | {d.get('_speed_str', 'N/A').strip()} | 已下载: {d.get('_downloaded_bytes_str', 'N/A')}", end='') if d['status'] == 'downloading' else None],
|
||||
}
|
||||
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
ydl.download([video_url])
|
||||
logger.info(f"📥 下载 {date_str} 完成")
|
||||
|
||||
# 使用ffmpeg提取音频
|
||||
logger.info(f"🎵 开始提取 {date_str} 的音频...")
|
||||
result = subprocess.run([
|
||||
"ffmpeg",
|
||||
"-i", mp4_path,
|
||||
"-c:a", "libmp3lame", # 明确指定MP3编码器
|
||||
"-q:a", "0",
|
||||
"-map", "a",
|
||||
mp3_path,
|
||||
"-y" # 覆盖已存在文件
|
||||
], check=True, stdout=None, stderr=None)
|
||||
logger.info(f"🎵 提取 {date_str} 音频完成")
|
||||
|
||||
logger.info(f"✅ 成功处理 {date_str}: {mp4_path}, {mp3_path}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 处理 {date_str} 时发生异常: {e}")
|
||||
|
||||
# 在get_all_video_links函数后添加调用代码
|
||||
def process_videos(start_date, end_date):
|
||||
"""
|
||||
处理指定日期范围内的所有视频
|
||||
"""
|
||||
video_urls = get_all_video_links(start_date, end_date)
|
||||
|
||||
for sub_url in video_urls:
|
||||
if sub_url: # 确保url不为空
|
||||
"""
|
||||
date_match = re.search(r'/(\d{4})/(\d{2})/(\d{2})/', url)
|
||||
if not date_match:
|
||||
print(f"❌ 无法从URL提取日期: {url}")
|
||||
return
|
||||
|
||||
year, month, day = date_match.groups()
|
||||
date_str = f"{year}{month}{day}"
|
||||
"""
|
||||
date_str= sub_url['date']
|
||||
url = sub_url['url']
|
||||
|
||||
# 创建保存目录
|
||||
download_dir = "/home/simon/myquant/djapi/api/video/xwlb_video"
|
||||
download_and_extract_audio(url,date_str,download_dir)
|
||||
print("=" * 80)
|
||||
# MP3文件路径
|
||||
mp3_path = os.path.join(download_dir, f"{date_str}.mp3")
|
||||
# 分析提示词
|
||||
# prompt = "请总结这段由中国中央电视台新闻联播音频转为文字的文本，理解其主要内容并提取其中的关键信息。"
|
||||
# 输出文件夹
|
||||
output_folder = "/home/simon/myquant/djapi/api/video/audio_processing"
|
||||
|
||||
# 处理长音频
|
||||
try:
|
||||
result = process_long_audio(mp3_path, output_folder,date_str)
|
||||
news_to_db(date_str)
|
||||
# 打印分析结果
|
||||
print("分析结果:\n")
|
||||
print(result)
|
||||
except Exception as e:
|
||||
print(f"处理失败: {e}")
|
||||
print("=" * 80)
|
||||
|
||||
|
||||
# ========================
|
||||
# 主程序执行
|
||||
# ========================
|
||||
if __name__ == "__main__":
|
||||
|
||||
import sys
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
# 检查命令行参数
|
||||
if len(sys.argv) < 2:
|
||||
print("用法: python getVideo5.py <start_date> <end_date>")
|
||||
print("日期格式: YYYYMMDD")
|
||||
sys.exit(1)
|
||||
|
||||
start_date = sys.argv[1]
|
||||
end_date = sys.argv[2] if len(sys.argv) > 2 and sys.argv[2] else start_date
|
||||
|
||||
# 检查日期格式
|
||||
date_pattern = r'^\d{8}$'
|
||||
if not re.match(date_pattern, start_date) or not re.match(date_pattern, end_date):
|
||||
print("错误: 日期格式必须为 YYYYMMDD")
|
||||
sys.exit(1)
|
||||
|
||||
# 检查日期有效性
|
||||
try:
|
||||
#start_dt = datetime.strptime(start_date, '%Y%m%d')
|
||||
#end_dt = datetime.strptime(end_date, '%Y%m%d')
|
||||
|
||||
if start_date > end_date:
|
||||
print(f"错误: start_date {start_date} 不能大于 end_date {end_date}")
|
||||
sys.exit(1)
|
||||
print("正在抓取央视《新闻联播》历史完整版视频链接...")
|
||||
print("=" * 80)
|
||||
process_videos(start_date, end_date)
|
||||
exit()
|
||||
except ValueError as e:
|
||||
print(f"错误: 无效日期 - {e}")
|
||||
sys.exit(1)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
from getVideo5 import process_videos
|
||||
from datetime import datetime
|
||||
|
||||
# 获取当日日期并格式化为yyyymmdd
|
||||
today = datetime.now().strftime("%Y%m%d")
|
||||
start_date = today
|
||||
end_date = today
|
||||
|
||||
# 执行process_videos方法
|
||||
process_videos(start_date, end_date)
|
||||
@@ -0,0 +1,75 @@
|
||||
"""
|
||||
xwlb_daily 表结构如下：
|
||||
+--------------+---------+------+-----+---------+----------------+
|
||||
| Field | Type | Null | Key | Default | Extra |
|
||||
+--------------+---------+------+-----+---------+----------------+
|
||||
| nid | int(11) | NO | PRI | NULL | auto_increment |
|
||||
| news_days | date | NO | | NULL | |
|
||||
| daily_sub_id | int(11) | NO | | NULL | |
|
||||
| news_raw | text | NO | | NULL | |
|
||||
| news_improve | text | NO | | NULL | |
|
||||
| news_title | text | NO | | NULL | |
|
||||
+--------------+---------+------+-----+---------+----------------+
|
||||
"""
|
||||
|
||||
from mysqlHandle import MySQLDB
|
||||
from getVideo5 import process_videos
|
||||
from datetime import datetime, timedelta
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def get_missing_dates(start_date, end_date):
|
||||
"""
|
||||
给定日期范围，查询xwlb_daily表中缺失的日期
|
||||
"""
|
||||
try:
|
||||
# 连接数据库
|
||||
db = MySQLDB()
|
||||
|
||||
# 查询指定日期范围内存在的所有日期
|
||||
result = db.query_data(
|
||||
table="xwlb_daily",
|
||||
columns="DISTINCT(news_days) as news_days",
|
||||
where="news_days BETWEEN %s AND %s order by news_days",
|
||||
params=(start_date, end_date)
|
||||
)
|
||||
# 获取所有存在的日期
|
||||
existing_dates = [row['news_days'] for row in result]
|
||||
|
||||
# 生成完整的日期范围
|
||||
start = datetime.strptime(start_date, '%Y-%m-%d').date()
|
||||
end = datetime.strptime(end_date, '%Y-%m-%d').date()
|
||||
|
||||
all_dates = []
|
||||
current_date = start
|
||||
while current_date <= end:
|
||||
all_dates.append(current_date)
|
||||
current_date = current_date + timedelta(days=1)
|
||||
|
||||
# 找出缺失的日期
|
||||
existing_set = set(existing_dates)
|
||||
missing_dates = [date.strftime('%Y%m%d') for date in all_dates if date not in existing_set]
|
||||
|
||||
logger.info(f"查询日期范围 {start_date} 到 {end_date}")
|
||||
logger.info(f"存在 {len(existing_dates)} 天数据，缺失 {len(missing_dates)} 天数据")
|
||||
logger.info(f"缺失日期: {missing_dates}")
|
||||
return missing_dates
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"查询缺失日期时出错: {str(e)}")
|
||||
return []
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试代码
|
||||
start_date = "2025-01-01"
|
||||
end_date = "2025-10-25"
|
||||
missing_dates=get_missing_dates(start_date, end_date)
|
||||
for date_str in missing_dates:
|
||||
logger.info(f"正在处理缺失日期: {date_str}")
|
||||
try:
|
||||
process_videos(date_str,date_str)
|
||||
logger.info(f"成功处理日期: {date_str}")
|
||||
except Exception as e:
|
||||
logger.error(f"处理日期 {date_str} 时出错: {str(e)}")
|
||||
@@ -0,0 +1,5 @@
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from utils.mysql_handler import MySQLDB # noqa: F401, E402 — video 模块独立运行，sys.path 方式导入
|
||||
@@ -0,0 +1,115 @@
|
||||
"""
|
||||
xwlb_daily 表结构如下：
|
||||
+--------------+---------+------+-----+---------+----------------+
|
||||
| Field | Type | Null | Key | Default | Extra |
|
||||
+--------------+---------+------+-----+---------+----------------+
|
||||
| nid | int(11) | NO | PRI | NULL | auto_increment |
|
||||
| news_days | date | NO | | NULL | |
|
||||
| daily_sub_id | int(11) | NO | | NULL | |
|
||||
| news_raw | text | NO | | NULL | |
|
||||
| news_improve | text | NO | | NULL | |
|
||||
| news_title | text | NO | | NULL | |
|
||||
+--------------+---------+------+-----+---------+----------------+
|
||||
xwlb_daily_ext 表结构如下：
|
||||
+--------------+--------------+------+-----+---------+----------------+
|
||||
| Field | Type | Null | Key | Default | Extra |
|
||||
+--------------+--------------+------+-----+---------+----------------+
|
||||
| extid | int(11) | NO | PRI | NULL | auto_increment |
|
||||
| news_date | date | NO | | NULL | |
|
||||
| sub_id | tinyint(4) | NO | | NULL | |
|
||||
| news_title | varchar(256) | NO | | NULL | |
|
||||
| news_content | text | NO | | NULL | |
|
||||
+--------------+--------------+------+-----+---------+----------------+
|
||||
|
||||
获取给定日期的所有news_improve字段内容，以daily_sub_id 顺序拼接为一个字符串返回。
|
||||
调用mysqlHandler中的方法执行SQL查询。
|
||||
"""
|
||||
from mysqlHandle import MySQLDB
|
||||
from deepseek import deepseek_text
|
||||
import json
|
||||
import logging
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def get_news_improve_by_date(target_date):
|
||||
"""
|
||||
获取指定日期的所有news_improve内容，按daily_sub_id顺序拼接
|
||||
|
||||
Args:
|
||||
target_date: 目标日期，格式为'YYYY-MM-DD'
|
||||
|
||||
Returns:
|
||||
str: 拼接后的字符串
|
||||
"""
|
||||
try:
|
||||
# 创建数据库连接对象
|
||||
db = MySQLDB()
|
||||
# 查询目标日期在xwlb_daily_ext表中的记录数量
|
||||
count_result = db.query_data(
|
||||
table="xwlb_daily_ext",
|
||||
columns="COUNT(*) as count",
|
||||
where="news_date = %s",
|
||||
params=(target_date,)
|
||||
)
|
||||
# 如果记录数量存在且大于5条，则返回空字符串
|
||||
if count_result and count_result[0]['count'] > 5:
|
||||
return None
|
||||
# 重新创建数据库连接对象，因为每次查询都会关闭连接
|
||||
db = MySQLDB()
|
||||
# 查询目标日期在xwlb_daily表中的news_improve字段，按daily_sub_id升序排列
|
||||
result = db.query_data(
|
||||
table="xwlb_daily",
|
||||
columns="news_improve",
|
||||
where="news_days = %s order by daily_sub_id ASC",
|
||||
params=(target_date,))
|
||||
# 如果查询结果不为空
|
||||
if result:
|
||||
# 将每条记录的news_improve字段用换行符连接成字符串
|
||||
combined_content = '\n'.join([row['news_improve'] for row in result])
|
||||
# 返回拼接后的字符串
|
||||
return combined_content
|
||||
# 查询结果为空时返回空字符串
|
||||
return None
|
||||
except Exception as e:
|
||||
#print(f"查询失败: {e}")
|
||||
logger.error(f"查询失败: {e}")
|
||||
return None
|
||||
|
||||
def news_to_db(target_date):
|
||||
result = get_news_improve_by_date(target_date)
|
||||
if result is None:
|
||||
logger.warning(f"日期 {target_date} 没有新闻内容或者已经存在处理后的记录。跳过")
|
||||
return None
|
||||
logger.info(f"日期 {target_date} 的新闻内容长度：{len(result)} 字符")
|
||||
|
||||
prompt= "###请根据下面新闻内容的文本逻辑 \n - 帮我分割成各个独立的新闻内容（注意：不要修改新闻本身，仅分割文本），并给每个新闻总结一个标题; \n - 如果遇到'国内快讯'、'国际快讯'或'联播快讯'，也请根据每个条快讯分割为一个新闻以及新闻标题； \n - 返回json格式。json格式包含：news_id，news_title，news_content; news_id从1开始递增。"
|
||||
try:
|
||||
response = deepseek_text(result, prompt)
|
||||
news_list = json.loads(response)
|
||||
db = MySQLDB()
|
||||
for news in news_list:
|
||||
db.insert_data(
|
||||
table="xwlb_daily_ext",
|
||||
data={
|
||||
"news_date": target_date,
|
||||
"sub_id": news["news_id"],
|
||||
"news_title": news["news_title"][:256], # 确保不超过varchar(256)限制
|
||||
"news_content": news["news_content"]
|
||||
}
|
||||
)
|
||||
logger.info(f"成功插入 {len(news_list)} 条新闻到数据库")
|
||||
except json.JSONDecodeError as e:
|
||||
logger.error(f"JSON解析失败: {e}")
|
||||
logger.error(f"DeepSeek API返回内容: {response}")
|
||||
except Exception as e:
|
||||
logger.error(f"插入数据库失败: {e}")
|
||||
#print(f"DeepSeek API返回结果：{response}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
from datetime import datetime
|
||||
#提供日期参数，格式：YYYY-MM-DD
|
||||
|
||||
target_date = datetime.now().strftime('%Y-%m-%d')
|
||||
news_to_db(target_date)
|
||||
@@ -0,0 +1,112 @@
|
||||
"""
|
||||
newsRedo — 手动重新执行新闻 AI 分割流程。
|
||||
|
||||
用法：
|
||||
python newsRedo.py # 默认当天日期
|
||||
python newsRedo.py 20250601 # yyyymmdd 格式
|
||||
python newsRedo.py 2025-06-01 # yyyy-mm-dd 格式
|
||||
|
||||
流程：
|
||||
1. 检查 xwlb_daily_ext 是否已有 >5 条 → 已处理过，正常跳过
|
||||
2. 检查 xwlb_daily 是否有当天记录 → 无记录则先跑 getVideo5 全流程
|
||||
3. 有记录但未处理 → 直接执行 news_to_db() AI 分割
|
||||
"""
|
||||
import sys
|
||||
import re
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from mysqlHandle import MySQLDB
|
||||
from newsProcess import news_to_db
|
||||
from getVideo5 import process_videos
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _parse_date(date_str):
|
||||
"""解析日期，返回 (yyyymmdd_str, yyyy_mm_dd_str)，或报错退出"""
|
||||
if not date_str:
|
||||
today = datetime.now()
|
||||
d8 = today.strftime('%Y%m%d')
|
||||
d10 = today.strftime('%Y-%m-%d')
|
||||
logger.info(f"未指定日期，使用当天: {d10}")
|
||||
return d8, d10
|
||||
|
||||
if re.match(r'^\d{4}-\d{2}-\d{2}$', date_str):
|
||||
try:
|
||||
datetime.strptime(date_str, '%Y-%m-%d')
|
||||
except ValueError:
|
||||
print(f"无效日期: {date_str}")
|
||||
sys.exit(1)
|
||||
return date_str.replace('-', ''), date_str
|
||||
|
||||
if re.match(r'^\d{8}$', date_str):
|
||||
try:
|
||||
datetime.strptime(date_str, '%Y%m%d')
|
||||
except ValueError:
|
||||
print(f"无效日期: {date_str}")
|
||||
sys.exit(1)
|
||||
return date_str, f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:8]}"
|
||||
|
||||
print("日期格式错误，请使用 yyyymmdd 或 yyyy-mm-dd 格式")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
date_str = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
date_d8, date_d10 = _parse_date(date_str)
|
||||
|
||||
db = MySQLDB()
|
||||
|
||||
# 1. 检查 xwlb_daily_ext 是否已处理过
|
||||
try:
|
||||
ext_count = db.query_data(
|
||||
table="xwlb_daily_ext",
|
||||
columns="COUNT(*) as count",
|
||||
where="news_date = %s",
|
||||
params=(date_d10,)
|
||||
)
|
||||
if ext_count and ext_count[0]['count'] > 5:
|
||||
logger.info(f"日期 {date_d10} 已有 {ext_count[0]['count']} 条精编记录，无需重新处理。")
|
||||
return
|
||||
except Exception as e:
|
||||
logger.error(f"查询 xwlb_daily_ext 失败: {e}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
db = MySQLDB()
|
||||
|
||||
# 2. 检查 xwlb_daily 是否有当天数据
|
||||
try:
|
||||
daily_count = db.query_data(
|
||||
table="xwlb_daily",
|
||||
columns="COUNT(*) as count",
|
||||
where="news_days = %s",
|
||||
params=(date_d10,)
|
||||
)
|
||||
has_daily = daily_count and daily_count[0]['count'] > 0
|
||||
except Exception as e:
|
||||
logger.error(f"查询 xwlb_daily 失败: {e}")
|
||||
has_daily = False
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# 3. 分支处理
|
||||
if has_daily:
|
||||
logger.info(f"日期 {date_d10} 在 xwlb_daily 中有记录，直接执行 AI 分割。")
|
||||
try:
|
||||
news_to_db(date_d10)
|
||||
except Exception as e:
|
||||
logger.error(f"news_to_db 执行出错: {e}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
logger.info(f"日期 {date_d10} 在 xwlb_daily 中无记录，重新执行视频下载全流程。")
|
||||
try:
|
||||
process_videos(date_d8, date_d8)
|
||||
except Exception as e:
|
||||
logger.error(f"process_videos 执行出错: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,10 @@
|
||||
aiofiles==25.1.0
|
||||
aiohttp==3.12.15
|
||||
beautifulsoup4==4.14.2
|
||||
dashscope==1.24.6
|
||||
m3u8==6.0.0
|
||||
mysql_connector_repackaged==0.3.1
|
||||
playwright==1.55.0
|
||||
pydub==0.25.1
|
||||
Requests==2.32.5
|
||||
yt_dlp==2025.11.12
|
||||
@@ -0,0 +1,260 @@
|
||||
import sys
|
||||
from django.shortcuts import render
|
||||
from django.http import HttpResponse
|
||||
from rest_framework.decorators import api_view
|
||||
from rest_framework.response import Response
|
||||
from drf_spectacular.utils import extend_schema, OpenApiParameter, OpenApiTypes
|
||||
|
||||
from .stock.stock_utils import viewFunc_tsCodeAndDate, viewFunc_singleParam
|
||||
from .stock.stock_basic import getStockBasic, getStockListByIndustry, getStockInfo
|
||||
from .stock.getStockParam import getStockParam
|
||||
from .stock.getStockEp import getStockEp_ttm, get_quarterly_eps
|
||||
from .stock.getIndexs import get_index_daily_data, get_index_by_name
|
||||
from .stock.stockMargin import getStockMargin, getDailyMargin
|
||||
from .stock.getStockFina import get_finance_data_range
|
||||
from .stock.getStockDiv2 import analyze_stock_dividend_and_price
|
||||
from .stock.xwlbDaily import get_xwlb, get_xwlb_fine
|
||||
from .stock.getDivData_AK import get_akshare_dividend_yield
|
||||
from .serializers import (
|
||||
StockDailySerializer, StockInfoSerializer, IndustryStockSerializer,
|
||||
StockParamSerializer, StockEpSerializer, QuarterlyEpsSerializer,
|
||||
IndexInfoSerializer, IndexDailySerializer, MarginDailySerializer,
|
||||
StockMarginSerializer, FinanceDataSerializer, DividendSerializer,
|
||||
XwlbNewsSerializer,
|
||||
)
|
||||
|
||||
# === 通用参数定义（复用） ===
|
||||
_PARAM_TSCODE = OpenApiParameter(name='tscode', type=str, default='000001.SZ',
|
||||
description='股票代码，如 000001.SZ')
|
||||
_PARAM_INDEX_CODE = OpenApiParameter(name='tscode', type=str, default='000001.SH',
|
||||
description='指数代码，如 000001.SH=上证指数, 399001.SZ=深证成指, 399006.SZ=创业板指')
|
||||
_PARAM_START = OpenApiParameter(name='start_date', type=str, default='20200101',
|
||||
description='起始日期 yyyyMMdd')
|
||||
_PARAM_END = OpenApiParameter(name='end_date', type=str, default='20251231',
|
||||
description='结束日期 yyyyMMdd')
|
||||
_PARAM_INDEX_NAME = OpenApiParameter(name='index_name', type=str, default='沪深300',
|
||||
description='指数名称，如 沪深300、上证50')
|
||||
_PARAM_INDUSTRY = OpenApiParameter(name='industry', type=str, required=True,
|
||||
description='行业名称，如 银行、半导体')
|
||||
_PARAM_TRADE_DATE = OpenApiParameter(name='trade_date', type=str,
|
||||
description='交易日期 yyyyMMdd')
|
||||
_PARAM_EXCHANGE_ID = OpenApiParameter(name='exchange_id', type=str,
|
||||
description='交易所代码 SSE/SZSE')
|
||||
|
||||
|
||||
@extend_schema(
|
||||
responses={200: OpenApiTypes.STR},
|
||||
description='返回服务器 Python 版本',
|
||||
tags=['系统'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def python_version(request):
|
||||
return HttpResponse(f"Python Version: {sys.version}")
|
||||
|
||||
|
||||
@extend_schema(exclude=True)
|
||||
@api_view(['GET'])
|
||||
def home(request):
|
||||
return render(request, 'home.html')
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TSCODE, _PARAM_START, _PARAM_END],
|
||||
responses={200: StockDailySerializer(many=True)},
|
||||
description='获取个股日线行情数据（开高低收、成交量、成交额）',
|
||||
tags=['行情'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def stockbasic(request):
|
||||
return viewFunc_tsCodeAndDate(request, getStockBasic)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_INDUSTRY],
|
||||
responses={200: IndustryStockSerializer(many=True)},
|
||||
description='按申万行业分类查询成分股列表',
|
||||
tags=['基础数据'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def industrys(request):
|
||||
return viewFunc_singleParam(request, getStockListByIndustry, param_name='industry')
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TSCODE],
|
||||
responses={200: StockInfoSerializer()},
|
||||
description='获取个股基本信息（名称、行业、上市日期、交易所等）',
|
||||
tags=['基础数据'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def stockInfo(request):
|
||||
return viewFunc_singleParam(request, getStockInfo, param_name='tscode')
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TSCODE, _PARAM_START, _PARAM_END],
|
||||
responses={200: StockParamSerializer(many=True)},
|
||||
description='获取个股每日参数（市值、PE/PB/PS、换手率等）',
|
||||
tags=['行情'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def stockparam(request):
|
||||
return viewFunc_tsCodeAndDate(request, getStockParam)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TSCODE, _PARAM_START, _PARAM_END],
|
||||
responses={200: StockEpSerializer(many=True)},
|
||||
description='获取个股 TTM 每股收益（EPS）',
|
||||
tags=['财务'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def stockep(request):
|
||||
return viewFunc_tsCodeAndDate(request, getStockEp_ttm)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TSCODE, _PARAM_START, _PARAM_END],
|
||||
responses={200: QuarterlyEpsSerializer(many=True)},
|
||||
description='获取个股季度每股收益（EPS），按财报日期对齐',
|
||||
tags=['财务'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def quarterlyEps(request):
|
||||
return viewFunc_tsCodeAndDate(request, get_quarterly_eps)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_INDEX_NAME],
|
||||
responses={200: IndexInfoSerializer()},
|
||||
description='按名称模糊查询指数基本信息',
|
||||
tags=['指数'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def indexByName(request):
|
||||
index_name = request.GET.get('index_name', '沪深300')
|
||||
if not index_name:
|
||||
return Response({'error': '缺少 index_name 参数'}, status=400)
|
||||
try:
|
||||
data = get_index_by_name(index_name)
|
||||
dict_data = data.to_dict(orient='records')
|
||||
return Response(dict_data)
|
||||
except ImportError:
|
||||
return Response({'error': '模块不存在'}, status=500)
|
||||
except Exception as e:
|
||||
return Response({'error': str(e)}, status=500)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_INDEX_CODE, _PARAM_START, _PARAM_END],
|
||||
responses={200: IndexDailySerializer(many=True)},
|
||||
description='获取指数日线行情数据（含 PE/PB/市值/换手率等扩展指标）',
|
||||
tags=['指数'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def indexDatas(request):
|
||||
return viewFunc_tsCodeAndDate(request, get_index_daily_data)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TSCODE, _PARAM_START, _PARAM_END],
|
||||
responses={200: StockMarginSerializer(many=True)},
|
||||
description='获取个股融资融券明细数据',
|
||||
tags=['融资融券'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def stockMargin(request):
|
||||
return viewFunc_tsCodeAndDate(request, getStockMargin)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TRADE_DATE, _PARAM_START, _PARAM_END, _PARAM_EXCHANGE_ID],
|
||||
responses={200: MarginDailySerializer(many=True)},
|
||||
description='获取每日融资融券汇总数据（按交易所）',
|
||||
tags=['融资融券'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def dailyMargin(request):
|
||||
trade_date = request.GET.get('trade_date', None)
|
||||
start_date = request.GET.get('start_date', None)
|
||||
end_date = request.GET.get('end_date', None)
|
||||
exchange_id = request.GET.get('exchange_id', None)
|
||||
|
||||
try:
|
||||
data = getDailyMargin(trade_date=trade_date, start_date=start_date,
|
||||
end_date=end_date, exchange_id=exchange_id)
|
||||
dict_data = data.to_dict(orient='records')
|
||||
return Response(dict_data)
|
||||
except ImportError:
|
||||
return Response({'error': '模块不存在'}, status=500)
|
||||
except Exception as e:
|
||||
return Response({'error': str(e)}, status=500)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TSCODE, _PARAM_START, _PARAM_END],
|
||||
responses={200: FinanceDataSerializer(many=True)},
|
||||
description='获取个股财务报表分析数据（资产负债表+利润表+现金流，含运营/资产/负债/回报率指标）',
|
||||
tags=['财务'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def getFinaData(request):
|
||||
return viewFunc_tsCodeAndDate(request, get_finance_data_range)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TSCODE, _PARAM_START, _PARAM_END],
|
||||
responses={200: DividendSerializer(many=True)},
|
||||
description='获取个股股息率数据（含 TTM 分红、收盘价、股息率）',
|
||||
tags=['分红'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def getDivData(request):
|
||||
return viewFunc_tsCodeAndDate(request, analyze_stock_dividend_and_price)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_TSCODE, _PARAM_START, _PARAM_END],
|
||||
responses={200: DividendSerializer(many=True)},
|
||||
description='获取个股股息率数据（akshare 数据源，无需 token）',
|
||||
tags=['分红'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def getDivDataAkshare(request):
|
||||
return viewFunc_tsCodeAndDate(request, get_akshare_dividend_yield)
|
||||
|
||||
|
||||
def _xwlb_view(request, data_func):
|
||||
start_date = request.GET.get('start_date', None)
|
||||
end_date = request.GET.get('end_date', None)
|
||||
try:
|
||||
data = data_func(start_date=start_date, end_date=end_date)
|
||||
dict_data = data.to_dict(orient='records')
|
||||
status = "success" if dict_data else "Error"
|
||||
return Response({"status": status, "data": {"news": dict_data}})
|
||||
except ImportError:
|
||||
return Response({'error': '模块不存在'}, status=500)
|
||||
except Exception as e:
|
||||
return Response({'error': str(e)}, status=500)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_START, _PARAM_END],
|
||||
responses={200: XwlbNewsSerializer(many=True)},
|
||||
description='获取新闻联播原始识别文本（ASR 转写结果）',
|
||||
tags=['新闻联播'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def xwlbNews(request):
|
||||
return _xwlb_view(request, get_xwlb)
|
||||
|
||||
|
||||
@extend_schema(
|
||||
parameters=[_PARAM_START, _PARAM_END],
|
||||
responses={200: XwlbNewsSerializer(many=True)},
|
||||
description='获取新闻联播精编内容（AI 分割+标题提取后的独立新闻）',
|
||||
tags=['新闻联播'],
|
||||
)
|
||||
@api_view(['GET'])
|
||||
def xwlbFine(request):
|
||||
return _xwlb_view(request, get_xwlb_fine)
|
||||
Reference in New Issue
Block a user