simonandClaude Opus 4.7
271a9343a5
7 Sprints 全部完成: Sprint 0: 基础设施 (DataManager + MariaDB) Sprint 1: 因子引擎 (34因子/12分类) Sprint 2: VectorBT 回测 (5策略+截面) Sprint 3: Optuna 优化 (+Walk-Forward) Sprint 4: ML 模型 (LightGBM+CatBoost) Sprint 5: Qwen 情绪因子 (三源新闻+日期对齐) Sprint 6: Agent 系统 (4Agent+日报.md/.html) 生产加固 (15项): Tushare双源fallback, SSH自动恢复, pool_pre_ping, save_daily先删后插, load_dotenv绝对路径, 日报5d/20d修复, RiskAgent改上证指数, 昨日对比+数据截止, mac_report utf8mb4, CLAUDE-*.md 9条已知Bug, demo全参数化, djapi数据源归一化, indexDatas API修正 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
76 lines
2.9 KiB
Python
76 lines
2.9 KiB
Python
"""
|
|
xwlb_daily 表结构如下：
|
|
+--------------+---------+------+-----+---------+----------------+
|
|
| Field | Type | Null | Key | Default | Extra |
|
|
+--------------+---------+------+-----+---------+----------------+
|
|
| nid | int(11) | NO | PRI | NULL | auto_increment |
|
|
| news_days | date | NO | | NULL | |
|
|
| daily_sub_id | int(11) | NO | | NULL | |
|
|
| news_raw | text | NO | | NULL | |
|
|
| news_improve | text | NO | | NULL | |
|
|
| news_title | text | NO | | NULL | |
|
|
+--------------+---------+------+-----+---------+----------------+
|
|
"""
|
|
|
|
from mysqlHandle import MySQLDB
|
|
from getVideo5 import process_videos
|
|
from datetime import datetime, timedelta
|
|
import logging
|
|
|
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def get_missing_dates(start_date, end_date):
|
|
"""
|
|
给定日期范围，查询xwlb_daily表中缺失的日期
|
|
"""
|
|
try:
|
|
# 连接数据库
|
|
db = MySQLDB()
|
|
|
|
# 查询指定日期范围内存在的所有日期
|
|
result = db.query_data(
|
|
table="xwlb_daily",
|
|
columns="DISTINCT(news_days) as news_days",
|
|
where="news_days BETWEEN %s AND %s order by news_days",
|
|
params=(start_date, end_date)
|
|
)
|
|
# 获取所有存在的日期
|
|
existing_dates = [row['news_days'] for row in result]
|
|
|
|
# 生成完整的日期范围
|
|
start = datetime.strptime(start_date, '%Y-%m-%d').date()
|
|
end = datetime.strptime(end_date, '%Y-%m-%d').date()
|
|
|
|
all_dates = []
|
|
current_date = start
|
|
while current_date <= end:
|
|
all_dates.append(current_date)
|
|
current_date = current_date + timedelta(days=1)
|
|
|
|
# 找出缺失的日期
|
|
existing_set = set(existing_dates)
|
|
missing_dates = [date.strftime('%Y%m%d') for date in all_dates if date not in existing_set]
|
|
|
|
logger.info(f"查询日期范围 {start_date} 到 {end_date}")
|
|
logger.info(f"存在 {len(existing_dates)} 天数据，缺失 {len(missing_dates)} 天数据")
|
|
logger.info(f"缺失日期: {missing_dates}")
|
|
return missing_dates
|
|
|
|
except Exception as e:
|
|
logger.error(f"查询缺失日期时出错: {str(e)}")
|
|
return []
|
|
|
|
if __name__ == "__main__":
|
|
# 测试代码
|
|
start_date = "2025-01-01"
|
|
end_date = "2025-10-25"
|
|
missing_dates=get_missing_dates(start_date, end_date)
|
|
for date_str in missing_dates:
|
|
logger.info(f"正在处理缺失日期: {date_str}")
|
|
try:
|
|
process_videos(date_str,date_str)
|
|
logger.info(f"成功处理日期: {date_str}")
|
|
except Exception as e:
|
|
logger.error(f"处理日期 {date_str} 时出错: {str(e)}")
|