"""对比 news_raw / news_improve（并联动 news_content），评估 LLM 校对的必要性与 token 成本

    .venv/bin/python tools/compare_raw_improve.py [--limit N] [--examples N] [--md 输出文件]

分类口径（逐分片）：
    identical     原文与校对后完全相同 → 校对没做任何事
    punct_only    去掉标点/空白后相同 → 校对只动了标点、空白、断句
    numeral_only  再去掉数字（中文数字与阿拉伯数字）后相同 → 只动了数字写法或标点
    word_change   仍有差异 → 真正改了字词（可能修错别字，也可能是改写/篡改）

token 估算：中文近似 1 字符 ≈ 1 token（保守上限，实际分词器约 0.6~1.0），
只用于比较两个方案的**相对**开销，绝对费用请按自己供应商单价换算。
"""
import argparse
import re
import sys
from collections import Counter
from datetime import date
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from mysqlHandle import MySQLDB  # noqa: E402

# 标点/空白（除中文、字母、数字之外的一切）
_NON_WORD = re.compile(r'[^\u4e00-\u9fffA-Za-z0-9]')
# 中文数字（与阿拉伯数字一并视为"数字"）
_CN_NUM = '零〇一二两三四五六七八九十百千万亿'
_NON_WORD_OR_DIGIT = re.compile(r'[^\u4e00-\u9fffA-Za-z]')
_PUNCT_ONLY = re.compile(r'[\u4e00-\u9fffA-Za-z0-9]')


def strip_punct(text):
    """去掉标点与空白，只留中文/字母/数字"""
    return _PUNCT_ONLY.findall(text or '')


def strip_punct_and_digits(text):
    """去掉标点、空白与所有数字（含中文数字）"""
    return [ch for ch in (text or '') if ch not in _CN_NUM and not ch.isdigit() and _PUNCT_ONLY.match(ch)]


def punct_density(text):
    """每 100 字符中的标点数"""
    text = text or ''
    if not text:
        return 0.0
    return len(_NON_WORD.findall(text)) / len(text) * 100


def title_marks(text):
    """书名号《》出现次数（可读性的直观指标）"""
    return (text or '').count('《')


def multiset_delta(a, b):
    """a 相对 b 的字符多重集差异数（近似"改了多少字"，O(n)）"""
    ca, cb = Counter(a or ''), Counter(b or '')
    return sum((ca - cb).values())


def classify(raw, improve):
    if raw == improve:
        return 'identical'
    if strip_punct(raw) == strip_punct(improve):
        return 'punct_only'
    if strip_punct_and_digits(raw) == strip_punct_and_digits(improve):
        return 'numeral_only'
    return 'word_change'


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--limit', type=int, default=0, help='只取最近 N 天')
    ap.add_argument('--examples', type=int, default=2, help='每类抽样条数')
    ap.add_argument('--md', default='', help='同时写出 markdown 报告')
    args = ap.parse_args()

    db = MySQLDB()
    try:
        where = '1=1'
        params = None
        if args.limit:
            dates = db.query_data(
                'xwlb_daily', 'DISTINCT news_days d',
                '1=1 ORDER BY news_days DESC LIMIT %s', (args.limit,))
            if not dates:
                print('没有数据')
                return 1
            d0 = min(r['d'] for r in dates)
            where, params = 'news_days >= %s', (d0,)
        rows = db.query_data(
            'xwlb_daily', 'news_days, daily_sub_id, news_raw, news_improve',
            f'{where} ORDER BY news_days, daily_sub_id', params)
        content_rows = db.query_data(
            'xwlb_daily_ext', 'news_date, sub_id, news_content',
            '1=1 ORDER BY news_date, sub_id', None)
    finally:
        db.close()

    # ---------- 逐分片分类 ----------
    stats = {k: {'n': 0, 'len_raw': 0, 'len_imp': 0, 'delta': 0, 'delta_words': 0}
             for k in ('identical', 'punct_only', 'numeral_only', 'word_change')}
    removed_chars, added_chars = Counter(), Counter()
    monthly = {}
    examples = {k: [] for k in stats}
    total_raw = total_imp = 0
    pd_raw = pd_imp = 0.0
    tm_raw = tm_imp = 0
    days = set()
    day_changed = {}
    for r in rows:
        raw, imp = r['news_raw'] or '', r['news_improve'] or ''
        cat = classify(raw, imp)
        s = stats[cat]
        s['n'] += 1
        s['len_raw'] += len(raw)
        s['len_imp'] += len(imp)
        s['delta'] += multiset_delta(raw, imp)
        # 排除标点/数字后的"真实字词"改动量
        s['delta_words'] += multiset_delta(''.join(strip_punct_and_digits(raw)),
                                           ''.join(strip_punct_and_digits(imp)))
        if cat == 'word_change':
            removed_chars.update((Counter(raw) - Counter(imp)).elements())
            added_chars.update((Counter(imp) - Counter(raw)).elements())
        key = str(r['news_days'])[:7]
        m = monthly.setdefault(key, {'n': 0, 'same': 0})
        m['n'] += 1
        if cat == 'identical':
            m['same'] += 1
        total_raw += len(raw)
        total_imp += len(imp)
        pd_raw += punct_density(raw)
        pd_imp += punct_density(imp)
        tm_raw += title_marks(raw)
        tm_imp += title_marks(imp)
        days.add(r['news_days'])
        day_changed.setdefault(r['news_days'], 0)
        if cat != 'identical':
            day_changed[r['news_days']] += 1
        if len(examples[cat]) < args.examples:
            examples[cat].append((r['news_days'], r['daily_sub_id'], raw, imp))

    n = len(rows) or 1
    out = []
    def w(line=''):
        print(line)
        out.append(line)

    w(f"# news_raw vs news_improve 评估报告")
    w()
    w(f"样本：`xwlb_daily` {len(rows)} 个分片，覆盖 {len(days)} 天"
      f"（{min(days)} ~ {max(days)}）；`xwlb_daily_ext` {len(content_rows)} 条")
    w()
    w("## 1. 校对到底改了什么")
    w()
    w("| 分类 | 分片数 | 占比 | 平均改动字符数 | 含义 |")
    w("|---|---|---|---|---|")
    label = {
        'identical': '完全没改',
        'punct_only': '只改标点/空白/断句',
        'numeral_only': '只改数字写法或标点',
        'word_change': '**真的改了字词**',
    }
    for k in ('identical', 'punct_only', 'numeral_only', 'word_change'):
        s = stats[k]
        avg_delta = s['delta'] / s['n'] if s['n'] else 0
        w(f"| {k} | {s['n']} | {s['n'] / n * 100:.1f}% | {avg_delta:.1f} | {label[k]} |")

    wc = stats['word_change']
    if wc['n']:
        w()
        w(f"「真的改了字词」的 {wc['n']} 个分片里，**排除标点与数字后**的平均改动量只有 "
          f"**{wc['delta_words'] / wc['n']:.1f} 字**"
          f"（占分片平均长度 {wc['len_raw'] / wc['n']:.0f} 字的 "
          f"{wc['delta_words'] / max(wc['len_raw'], 1) * 100:.1f}%）")
        w()
        w("改动最频繁的字符（raw 有而 improve 没有 → improve 新增）:")
        w()
        w("| 被替换掉的字符 | 次数 | 新增的字符 | 次数 |")
        w("|---|---|---|---|")
        top_rm = removed_chars.most_common(10)
        top_ad = added_chars.most_common(10)
        for i in range(max(len(top_rm), len(top_ad))):
            rm = top_rm[i] if i < len(top_rm) else ('', '')
            ad = top_ad[i] if i < len(top_ad) else ('', '')
            rm_ch = rm[0].replace(' ', '␠').replace('\n', '⏎')
            ad_ch = ad[0].replace(' ', '␠').replace('\n', '⏎')
            w(f"| `{rm_ch}` | {rm[1]} | `{ad_ch}` | {ad[1]} |")

    changed = n - stats['identical']['n']
    w()
    w(f"- 校对**实际生效**的分片：{changed}/{n}（{changed / n * 100:.1f}%）；"
      f"完全没动的：{stats['identical']['n']}/{n}（{stats['identical']['n'] / n * 100:.1f}%）")
    all_same_days = sum(1 for d in days if day_changed.get(d, 0) == 0)
    w(f"- 全天 11 个分片**全都没被改**的天数：{all_same_days}/{len(days)}")
    w()
    w("### 可读性指标（越大越接近正式书面语）")
    w()
    w("| 指标 | 原文 news_raw | 校对后 news_improve | 变化 |")
    w("|---|---|---|---|")
    w(f"| 标点密度（每百字标点数） | {pd_raw / n:.2f} | {pd_imp / n:.2f} | "
      f"{(pd_imp - pd_raw) / n:+.2f} |")
    w(f"| 书名号《总数 | {tm_raw} | {tm_imp} | {tm_imp - tm_raw:+d} |")
    w(f"| 总字数 | {total_raw} | {total_imp} | {total_imp - total_raw:+d} |")

    if stats['word_change']['n']:
        w()
        w("### 「真的改了字词」的抽样（判断是修错字还是改写）")
        for d, sid, raw, imp in examples['word_change']:
            w()
            w(f"- **{d} 第{sid}片**（改动约 {multiset_delta(raw, imp)} 字）")
            w(f"  - raw: {raw[:120]}")
            w(f"  - imp: {imp[:120]}")

    w()
    w("### 按月的「完全没改」比例（用于识别校对失效的历史区间）")
    w()
    w("| 月份 | 分片数 | 完全没改 | 占比 |")
    w("|---|---|---|---|")
    for key in sorted(monthly):
        m = monthly[key]
        w(f"| {key} | {m['n']} | {m['same']} | {m['same'] / m['n'] * 100:.0f}% |")

    # ---------- token 预算 ----------
    w()
    w("## 2. token 开销：现状 vs 两个替代方案")
    w()
    by_day_raw, by_day_imp = {}, {}
    for r in rows:
        by_day_raw[r['news_days']] = by_day_raw.get(r['news_days'], 0) + len(r['news_raw'] or '')
        by_day_imp[r['news_days']] = by_day_imp.get(r['news_days'], 0) + len(r['news_improve'] or '')
    by_day_content = {}
    for r in content_rows:
        if r['news_date'] in by_day_raw:
            by_day_content[r['news_date']] = by_day_content.get(r['news_date'], 0) + len(r['news_content'] or '')

    common = sorted(set(by_day_raw) & set(by_day_content))
    if common:
        raw_c = sum(by_day_raw[d] for d in common) / len(common)
        imp_c = sum(by_day_imp[d] for d in common) / len(common)
        con_c = sum(by_day_content[d] for d in common) / len(common)
        corr_in, corr_out = raw_c, imp_c
        split_in, split_out = imp_c, con_c
        merged_in, merged_out = raw_c, con_c
        cur = corr_in + corr_out + split_in + split_out
        merged = merged_in + merged_out
        off = raw_c + con_c
        w(f"统计口径：{len(common)} 天同时有原文与精编的天数，取日均字符数（≈token 数，中文 1 字≈1 token）")
        w()
        w("| 方案 | 输入 | 输出 | 合计/天 | 相对现状 |")
        w("|---|---|---|---|---|")
        w(f"| A 现状（校对 + 切分 两次调用） | {corr_in + split_in:.0f} | {corr_out + split_out:.0f} | **{cur:.0f}** | — |")
        w(f"| B 合并进切分（一次调用同时校对+切分） | {merged_in:.0f} | {merged_out:.0f} | **{merged:.0f}** | "
          f"省 {(1 - merged / cur) * 100:.0f}% |")
        w(f"| C 关掉校对（直接用 ASR 原文切分） | {off:.0f} | {con_c:.0f} | **{off:.0f}** | "
          f"省 {(1 - off / cur) * 100:.0f}% |")
        w()
        w(f"其中校对这一次调用本身消耗 输入 {corr_in:.0f} + 输出 {corr_out:.0f} = "
          f"**{corr_in + corr_out:.0f} token/天**，占现状总开销的 "
          f"{(corr_in + corr_out) / cur * 100:.0f}%。")
        w()
        w(f"按 {len(common)} 天累计：校对一项约消耗 "
          f"{(corr_in + corr_out) * len(common) / 1e6:.2f} M token。")

    if args.md:
        Path(args.md).write_text('\n'.join(out) + '\n', encoding='utf-8')
        print(f"\n报告已写入 {args.md}")
    return 0


if __name__ == '__main__':
    sys.exit(main())