"""中间产物清理 用法: python cleanup.py # 按 config.yml 的 cleanup 配置清理 python cleanup.py --dry-run # 只列出将删除的文件,不真删 清理范围(由 config.yml 的 cleanup 段控制): paths.video_dir 下所有 *.mp3 / *.mp4 paths.audio_dir 下所有 *.wav 当日流程(getVideo5.process_videos)在一天的任务成功结束后会自动调用本模块; 失败时默认保留文件以便重跑(cleanup.only_on_success: 1)。 """ import argparse import logging import config logger = logging.getLogger(__name__) def _list_files(directory, patterns): if not directory.is_dir(): return [] found = [] for pattern in patterns: found.extend(sorted(p for p in directory.glob(pattern) if p.is_file())) return found def collect_targets(): """返回 [(路径, 目录角色)],受 cleanup.remove_video / remove_audio 控制""" targets = [] if config.get_bool('cleanup.remove_video', True): targets += [(p, 'video') for p in _list_files(config.video_dir(), ('*.mp3', '*.mp4'))] if config.get_bool('cleanup.remove_audio', True): # 只删 wav。识别完成标记在 state/ 下,是长期凭证,**不能删** # (删了就分不清"已完成"与"识别只跑了一半",见 scripts/day_status.py) targets += [(p, 'audio') for p in _list_files(config.audio_dir(), ('*.wav',))] return targets def cleanup_intermediates(dry_run=False, force=False, reason=''): """ 删除中间产物;返回 {'removed': n, 'bytes': n, 'skipped': bool} 参数: dry_run: 只统计不删除 force: 忽略 cleanup.after_daily_run 开关(供手动调用) reason: 日志说明(一般是日期) """ if not force and not config.get_bool('cleanup.after_daily_run', True): logger.info("cleanup.after_daily_run 已关闭,跳过清理") return {'removed': 0, 'bytes': 0, 'skipped': True} targets = collect_targets() if not targets: logger.info("没有需要清理的中间产物(%s)", reason or '手动清理') return {'removed': 0, 'bytes': 0, 'skipped': False} removed = 0 freed = 0 for path, role in targets: try: size = path.stat().st_size except OSError: size = 0 if dry_run: logger.info("[dry-run] 将删除 %s(%.1f MB)", path, size / 1024 / 1024) removed += 1 freed += size continue try: path.unlink() removed += 1 freed += size logger.debug("已删除 %s", path) except OSError as e: logger.warning("删除失败 %s: %s", path, e) action = '将清理' if dry_run else '已清理' logger.info("%s中间产物 %d 个文件(释放 %.1f MB)%s", action, removed, freed / 1024 / 1024, f" | {reason}" if reason else '') return {'removed': removed, 'bytes': freed, 'skipped': False} def maybe_cleanup_after_run(date_str, day_ok): """ 当天流程结束后按 config.yml 的 cleanup 段决定是否清理 参数: date_str: 日期(仅用于日志) day_ok: 当天是否全部成功(识别分片全成功 + 切分已就绪) """ if not config.get_bool('cleanup.after_daily_run', True): logger.info("cleanup.after_daily_run 已关闭,跳过清理") return {'removed': 0, 'bytes': 0, 'skipped': True} if not day_ok and config.get_bool('cleanup.only_on_success', True): logger.warning("⚠️ %s 未全部成功,保留中间产物以便重跑" "(cleanup.only_on_success=1;改为 0 则无论成败都清理)", date_str) return {'removed': 0, 'bytes': 0, 'skipped': True} reason = f'{date_str} 任务完成' if day_ok else f'{date_str} 任务未全部成功(only_on_success=0)' return cleanup_intermediates(reason=reason) def main(): parser = argparse.ArgumentParser(description='清理 xwlb 中间产物(mp3/mp4/wav)') parser.add_argument('--dry-run', action='store_true', help='只列出将删除的文件') args = parser.parse_args() logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') config.log_summary() result = cleanup_intermediates(dry_run=args.dry_run, force=True, reason='手动执行') return 0 if result['removed'] >= 0 else 1 if __name__ == '__main__': raise SystemExit(main())