fix: 日报数据总览统计修复与口径说明

- _collect_pipeline_stats 24h 统计改用 fetched_at(抓取时间), 原实现从 URL 猜日期对多数源失效(实测 804 篇文章仅 3 条计入)
- 删除废弃的 _extract_date_from_url
- docs/db_schema.md 新增 3.1 口径说明: raw_total=当天抓取文章(sum=raw_by_source), raw_total_24h=24h 内抓取, news.total=30h 窗口 LLM 事件数(≠raw_total), importances/sentiments 之和=news.total
This commit is contained in:
2026-08-06 08:17:14 +08:00
parent 2f2428aa9a
commit 0c032196d2
2 changed files with 25 additions and 24 deletions
+11 -24
View File
@@ -414,27 +414,8 @@ def _load_article_urls_from_index(index_path: Path) -> list[dict]:
return articles
def _extract_date_from_url(url: str) -> datetime | None:
"""从 URL 中提取发布日期(用于估算 24h 新鲜度)。"""
import re as _re2
patterns = [
_re2.compile(r'/(\d{4})[-/](\d{2})[-/](\d{2})/'),
_re2.compile(r'/(\d{4})(\d{2})(\d{2})/'),
_re2.compile(r'(\d{4})(\d{2})(\d{2})\.(?:s?html|pdf)'),
_re2.compile(r'/t(\d{4})(\d{2})(\d{2})_'),
]
for pat in patterns:
m = pat.search(url)
if m:
try:
return datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)))
except ValueError:
pass
return None
def _collect_pipeline_stats(day_str: str) -> dict[str, Any]:
"""收集管道统计数据(仅计文章级条目 + 24h 新鲜度)。"""
"""收集管道统计数据(仅计文章级条目 + 24h 抓取新鲜度, 用 fetched_at)。"""
now = datetime.now()
cutoff_24h = now - timedelta(hours=24)
@@ -451,12 +432,18 @@ def _collect_pipeline_stats(day_str: str) -> dict[str, Any]:
raw_by_source[name] = n
raw_total += n
# 统计 24h 内文章
# 统计 24h 内抓取的文章(用 fetched_at;原实现从 URL 猜日期, 对多数源失效导致全 0)
n_24h = 0
for art in articles:
dt = _extract_date_from_url(art.get("url", ""))
if dt and dt >= cutoff_24h:
n_24h += 1
fa = art.get("fetched_at")
if fa:
try:
if datetime.fromisoformat(fa) >= cutoff_24h:
n_24h += 1
except ValueError:
n_24h += 1 # 时间格式异常时保守计入
else:
n_24h += 1 # 时间缺失时保守计入
raw_by_source_24h[name] = n_24h
raw_total_24h += n_24h