simon
60f8c263f2
从 CCTV 主页抓取《新闻联播》，下载 → 转 MP3/WAV → 静音切分 → ASR 识别 → LLM 校对 → 切分为单条新闻 → 入库 MySQL。 主要内容： - 全链路：getVideo5 抓取下载、audioRead 转写、deepseek 校对与切分、newsProcess 入库 - 可移植化：配置分层，.env 只放密钥、config.yml 放模型/接入点/路由/参数 - 可换供应商：endpoints（kind/base_url/api_key_env/extra_body）+ routes 按环节选路 - 数据保真：数值事实守卫，校对改动数字/年份/届次则整片回退 ASR 原文； 识别不完整不发布该日精编，避免半天内容被当成完整一天 - 定时任务：systemd 每天 21:00，失败 21:30 / 22:00 重试； 只缺切分时只重跑切分（省掉全部 ASR），用 state/.asr_complete_* 标记判定阶段 - 隧道自愈：13306 不通时自动执行 autossh.sh（所有入口共用，systemd 托管时只等待） - 中间产物每日清理；97 项离线自检（配置/清理/事实守卫/解析/隧道）
273 lines
11 KiB
Python
273 lines
11 KiB
Python
"""对比 news_raw / news_improve（并联动 news_content），评估 LLM 校对的必要性与 token 成本
|
||
|
||
.venv/bin/python tools/compare_raw_improve.py [--limit N] [--examples N] [--md 输出文件]
|
||
|
||
分类口径（逐分片）：
|
||
identical 原文与校对后完全相同 → 校对没做任何事
|
||
punct_only 去掉标点/空白后相同 → 校对只动了标点、空白、断句
|
||
numeral_only 再去掉数字（中文数字与阿拉伯数字）后相同 → 只动了数字写法或标点
|
||
word_change 仍有差异 → 真正改了字词（可能修错别字，也可能是改写/篡改）
|
||
|
||
token 估算：中文近似 1 字符 ≈ 1 token（保守上限，实际分词器约 0.6~1.0），
|
||
只用于比较两个方案的**相对**开销，绝对费用请按自己供应商单价换算。
|
||
"""
|
||
import argparse
|
||
import re
|
||
import sys
|
||
from collections import Counter
|
||
from datetime import date
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||
|
||
from mysqlHandle import MySQLDB # noqa: E402
|
||
|
||
# 标点/空白（除中文、字母、数字之外的一切）
|
||
_NON_WORD = re.compile(r'[^\u4e00-\u9fffA-Za-z0-9]')
|
||
# 中文数字（与阿拉伯数字一并视为"数字"）
|
||
_CN_NUM = '零〇一二两三四五六七八九十百千万亿'
|
||
_NON_WORD_OR_DIGIT = re.compile(r'[^\u4e00-\u9fffA-Za-z]')
|
||
_PUNCT_ONLY = re.compile(r'[\u4e00-\u9fffA-Za-z0-9]')
|
||
|
||
|
||
def strip_punct(text):
|
||
"""去掉标点与空白，只留中文/字母/数字"""
|
||
return _PUNCT_ONLY.findall(text or '')
|
||
|
||
|
||
def strip_punct_and_digits(text):
|
||
"""去掉标点、空白与所有数字（含中文数字）"""
|
||
return [ch for ch in (text or '') if ch not in _CN_NUM and not ch.isdigit() and _PUNCT_ONLY.match(ch)]
|
||
|
||
|
||
def punct_density(text):
|
||
"""每 100 字符中的标点数"""
|
||
text = text or ''
|
||
if not text:
|
||
return 0.0
|
||
return len(_NON_WORD.findall(text)) / len(text) * 100
|
||
|
||
|
||
def title_marks(text):
|
||
"""书名号《》出现次数（可读性的直观指标）"""
|
||
return (text or '').count('《')
|
||
|
||
|
||
def multiset_delta(a, b):
|
||
"""a 相对 b 的字符多重集差异数（近似"改了多少字"，O(n)）"""
|
||
ca, cb = Counter(a or ''), Counter(b or '')
|
||
return sum((ca - cb).values())
|
||
|
||
|
||
def classify(raw, improve):
|
||
if raw == improve:
|
||
return 'identical'
|
||
if strip_punct(raw) == strip_punct(improve):
|
||
return 'punct_only'
|
||
if strip_punct_and_digits(raw) == strip_punct_and_digits(improve):
|
||
return 'numeral_only'
|
||
return 'word_change'
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument('--limit', type=int, default=0, help='只取最近 N 天')
|
||
ap.add_argument('--examples', type=int, default=2, help='每类抽样条数')
|
||
ap.add_argument('--md', default='', help='同时写出 markdown 报告')
|
||
args = ap.parse_args()
|
||
|
||
db = MySQLDB()
|
||
try:
|
||
where = '1=1'
|
||
params = None
|
||
if args.limit:
|
||
dates = db.query_data(
|
||
'xwlb_daily', 'DISTINCT news_days d',
|
||
'1=1 ORDER BY news_days DESC LIMIT %s', (args.limit,))
|
||
if not dates:
|
||
print('没有数据')
|
||
return 1
|
||
d0 = min(r['d'] for r in dates)
|
||
where, params = 'news_days >= %s', (d0,)
|
||
rows = db.query_data(
|
||
'xwlb_daily', 'news_days, daily_sub_id, news_raw, news_improve',
|
||
f'{where} ORDER BY news_days, daily_sub_id', params)
|
||
content_rows = db.query_data(
|
||
'xwlb_daily_ext', 'news_date, sub_id, news_content',
|
||
'1=1 ORDER BY news_date, sub_id', None)
|
||
finally:
|
||
db.close()
|
||
|
||
# ---------- 逐分片分类 ----------
|
||
stats = {k: {'n': 0, 'len_raw': 0, 'len_imp': 0, 'delta': 0, 'delta_words': 0}
|
||
for k in ('identical', 'punct_only', 'numeral_only', 'word_change')}
|
||
removed_chars, added_chars = Counter(), Counter()
|
||
monthly = {}
|
||
examples = {k: [] for k in stats}
|
||
total_raw = total_imp = 0
|
||
pd_raw = pd_imp = 0.0
|
||
tm_raw = tm_imp = 0
|
||
days = set()
|
||
day_changed = {}
|
||
for r in rows:
|
||
raw, imp = r['news_raw'] or '', r['news_improve'] or ''
|
||
cat = classify(raw, imp)
|
||
s = stats[cat]
|
||
s['n'] += 1
|
||
s['len_raw'] += len(raw)
|
||
s['len_imp'] += len(imp)
|
||
s['delta'] += multiset_delta(raw, imp)
|
||
# 排除标点/数字后的"真实字词"改动量
|
||
s['delta_words'] += multiset_delta(''.join(strip_punct_and_digits(raw)),
|
||
''.join(strip_punct_and_digits(imp)))
|
||
if cat == 'word_change':
|
||
removed_chars.update((Counter(raw) - Counter(imp)).elements())
|
||
added_chars.update((Counter(imp) - Counter(raw)).elements())
|
||
key = str(r['news_days'])[:7]
|
||
m = monthly.setdefault(key, {'n': 0, 'same': 0})
|
||
m['n'] += 1
|
||
if cat == 'identical':
|
||
m['same'] += 1
|
||
total_raw += len(raw)
|
||
total_imp += len(imp)
|
||
pd_raw += punct_density(raw)
|
||
pd_imp += punct_density(imp)
|
||
tm_raw += title_marks(raw)
|
||
tm_imp += title_marks(imp)
|
||
days.add(r['news_days'])
|
||
day_changed.setdefault(r['news_days'], 0)
|
||
if cat != 'identical':
|
||
day_changed[r['news_days']] += 1
|
||
if len(examples[cat]) < args.examples:
|
||
examples[cat].append((r['news_days'], r['daily_sub_id'], raw, imp))
|
||
|
||
n = len(rows) or 1
|
||
out = []
|
||
def w(line=''):
|
||
print(line)
|
||
out.append(line)
|
||
|
||
w(f"# news_raw vs news_improve 评估报告")
|
||
w()
|
||
w(f"样本：`xwlb_daily` {len(rows)} 个分片，覆盖 {len(days)} 天"
|
||
f"（{min(days)} ~ {max(days)}）；`xwlb_daily_ext` {len(content_rows)} 条")
|
||
w()
|
||
w("## 1. 校对到底改了什么")
|
||
w()
|
||
w("| 分类 | 分片数 | 占比 | 平均改动字符数 | 含义 |")
|
||
w("|---|---|---|---|---|")
|
||
label = {
|
||
'identical': '完全没改',
|
||
'punct_only': '只改标点/空白/断句',
|
||
'numeral_only': '只改数字写法或标点',
|
||
'word_change': '**真的改了字词**',
|
||
}
|
||
for k in ('identical', 'punct_only', 'numeral_only', 'word_change'):
|
||
s = stats[k]
|
||
avg_delta = s['delta'] / s['n'] if s['n'] else 0
|
||
w(f"| {k} | {s['n']} | {s['n'] / n * 100:.1f}% | {avg_delta:.1f} | {label[k]} |")
|
||
|
||
wc = stats['word_change']
|
||
if wc['n']:
|
||
w()
|
||
w(f"「真的改了字词」的 {wc['n']} 个分片里，**排除标点与数字后**的平均改动量只有 "
|
||
f"**{wc['delta_words'] / wc['n']:.1f} 字**"
|
||
f"（占分片平均长度 {wc['len_raw'] / wc['n']:.0f} 字的 "
|
||
f"{wc['delta_words'] / max(wc['len_raw'], 1) * 100:.1f}%）")
|
||
w()
|
||
w("改动最频繁的字符（raw 有而 improve 没有 → improve 新增）:")
|
||
w()
|
||
w("| 被替换掉的字符 | 次数 | 新增的字符 | 次数 |")
|
||
w("|---|---|---|---|")
|
||
top_rm = removed_chars.most_common(10)
|
||
top_ad = added_chars.most_common(10)
|
||
for i in range(max(len(top_rm), len(top_ad))):
|
||
rm = top_rm[i] if i < len(top_rm) else ('', '')
|
||
ad = top_ad[i] if i < len(top_ad) else ('', '')
|
||
rm_ch = rm[0].replace(' ', '␠').replace('\n', '⏎')
|
||
ad_ch = ad[0].replace(' ', '␠').replace('\n', '⏎')
|
||
w(f"| `{rm_ch}` | {rm[1]} | `{ad_ch}` | {ad[1]} |")
|
||
|
||
changed = n - stats['identical']['n']
|
||
w()
|
||
w(f"- 校对**实际生效**的分片：{changed}/{n}（{changed / n * 100:.1f}%）；"
|
||
f"完全没动的：{stats['identical']['n']}/{n}（{stats['identical']['n'] / n * 100:.1f}%）")
|
||
all_same_days = sum(1 for d in days if day_changed.get(d, 0) == 0)
|
||
w(f"- 全天 11 个分片**全都没被改**的天数：{all_same_days}/{len(days)}")
|
||
w()
|
||
w("### 可读性指标（越大越接近正式书面语）")
|
||
w()
|
||
w("| 指标 | 原文 news_raw | 校对后 news_improve | 变化 |")
|
||
w("|---|---|---|---|")
|
||
w(f"| 标点密度（每百字标点数） | {pd_raw / n:.2f} | {pd_imp / n:.2f} | "
|
||
f"{(pd_imp - pd_raw) / n:+.2f} |")
|
||
w(f"| 书名号《总数 | {tm_raw} | {tm_imp} | {tm_imp - tm_raw:+d} |")
|
||
w(f"| 总字数 | {total_raw} | {total_imp} | {total_imp - total_raw:+d} |")
|
||
|
||
if stats['word_change']['n']:
|
||
w()
|
||
w("### 「真的改了字词」的抽样（判断是修错字还是改写）")
|
||
for d, sid, raw, imp in examples['word_change']:
|
||
w()
|
||
w(f"- **{d} 第{sid}片**（改动约 {multiset_delta(raw, imp)} 字）")
|
||
w(f" - raw: {raw[:120]}")
|
||
w(f" - imp: {imp[:120]}")
|
||
|
||
w()
|
||
w("### 按月的「完全没改」比例（用于识别校对失效的历史区间）")
|
||
w()
|
||
w("| 月份 | 分片数 | 完全没改 | 占比 |")
|
||
w("|---|---|---|---|")
|
||
for key in sorted(monthly):
|
||
m = monthly[key]
|
||
w(f"| {key} | {m['n']} | {m['same']} | {m['same'] / m['n'] * 100:.0f}% |")
|
||
|
||
# ---------- token 预算 ----------
|
||
w()
|
||
w("## 2. token 开销：现状 vs 两个替代方案")
|
||
w()
|
||
by_day_raw, by_day_imp = {}, {}
|
||
for r in rows:
|
||
by_day_raw[r['news_days']] = by_day_raw.get(r['news_days'], 0) + len(r['news_raw'] or '')
|
||
by_day_imp[r['news_days']] = by_day_imp.get(r['news_days'], 0) + len(r['news_improve'] or '')
|
||
by_day_content = {}
|
||
for r in content_rows:
|
||
if r['news_date'] in by_day_raw:
|
||
by_day_content[r['news_date']] = by_day_content.get(r['news_date'], 0) + len(r['news_content'] or '')
|
||
|
||
common = sorted(set(by_day_raw) & set(by_day_content))
|
||
if common:
|
||
raw_c = sum(by_day_raw[d] for d in common) / len(common)
|
||
imp_c = sum(by_day_imp[d] for d in common) / len(common)
|
||
con_c = sum(by_day_content[d] for d in common) / len(common)
|
||
corr_in, corr_out = raw_c, imp_c
|
||
split_in, split_out = imp_c, con_c
|
||
merged_in, merged_out = raw_c, con_c
|
||
cur = corr_in + corr_out + split_in + split_out
|
||
merged = merged_in + merged_out
|
||
off = raw_c + con_c
|
||
w(f"统计口径：{len(common)} 天同时有原文与精编的天数，取日均字符数（≈token 数，中文 1 字≈1 token）")
|
||
w()
|
||
w("| 方案 | 输入 | 输出 | 合计/天 | 相对现状 |")
|
||
w("|---|---|---|---|---|")
|
||
w(f"| A 现状（校对 + 切分 两次调用） | {corr_in + split_in:.0f} | {corr_out + split_out:.0f} | **{cur:.0f}** | — |")
|
||
w(f"| B 合并进切分（一次调用同时校对+切分） | {merged_in:.0f} | {merged_out:.0f} | **{merged:.0f}** | "
|
||
f"省 {(1 - merged / cur) * 100:.0f}% |")
|
||
w(f"| C 关掉校对（直接用 ASR 原文切分） | {off:.0f} | {con_c:.0f} | **{off:.0f}** | "
|
||
f"省 {(1 - off / cur) * 100:.0f}% |")
|
||
w()
|
||
w(f"其中校对这一次调用本身消耗 输入 {corr_in:.0f} + 输出 {corr_out:.0f} = "
|
||
f"**{corr_in + corr_out:.0f} token/天**，占现状总开销的 "
|
||
f"{(corr_in + corr_out) / cur * 100:.0f}%。")
|
||
w()
|
||
w(f"按 {len(common)} 天累计：校对一项约消耗 "
|
||
f"{(corr_in + corr_out) * len(common) / 1e6:.2f} M token。")
|
||
|
||
if args.md:
|
||
Path(args.md).write_text('\n'.join(out) + '\n', encoding='utf-8')
|
||
print(f"\n报告已写入 {args.md}")
|
||
return 0
|
||
|
||
|
||
if __name__ == '__main__':
|
||
sys.exit(main()) |