fix: 日报数据总览统计修复与口径说明
- _collect_pipeline_stats 24h 统计改用 fetched_at(抓取时间), 原实现从 URL 猜日期对多数源失效(实测 804 篇文章仅 3 条计入) - 删除废弃的 _extract_date_from_url - docs/db_schema.md 新增 3.1 口径说明: raw_total=当天抓取文章(sum=raw_by_source), raw_total_24h=24h 内抓取, news.total=30h 窗口 LLM 事件数(≠raw_total), importances/sentiments 之和=news.total
This commit is contained in:
+11
-24
@@ -414,27 +414,8 @@ def _load_article_urls_from_index(index_path: Path) -> list[dict]:
|
||||
return articles
|
||||
|
||||
|
||||
def _extract_date_from_url(url: str) -> datetime | None:
|
||||
"""从 URL 中提取发布日期(用于估算 24h 新鲜度)。"""
|
||||
import re as _re2
|
||||
patterns = [
|
||||
_re2.compile(r'/(\d{4})[-/](\d{2})[-/](\d{2})/'),
|
||||
_re2.compile(r'/(\d{4})(\d{2})(\d{2})/'),
|
||||
_re2.compile(r'(\d{4})(\d{2})(\d{2})\.(?:s?html|pdf)'),
|
||||
_re2.compile(r'/t(\d{4})(\d{2})(\d{2})_'),
|
||||
]
|
||||
for pat in patterns:
|
||||
m = pat.search(url)
|
||||
if m:
|
||||
try:
|
||||
return datetime(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _collect_pipeline_stats(day_str: str) -> dict[str, Any]:
|
||||
"""收集管道统计数据(仅计文章级条目 + 24h 新鲜度)。"""
|
||||
"""收集管道统计数据(仅计文章级条目 + 24h 抓取新鲜度, 用 fetched_at)。"""
|
||||
now = datetime.now()
|
||||
cutoff_24h = now - timedelta(hours=24)
|
||||
|
||||
@@ -451,12 +432,18 @@ def _collect_pipeline_stats(day_str: str) -> dict[str, Any]:
|
||||
raw_by_source[name] = n
|
||||
raw_total += n
|
||||
|
||||
# 统计 24h 内文章
|
||||
# 统计 24h 内抓取的文章(用 fetched_at;原实现从 URL 猜日期, 对多数源失效导致全 0)
|
||||
n_24h = 0
|
||||
for art in articles:
|
||||
dt = _extract_date_from_url(art.get("url", ""))
|
||||
if dt and dt >= cutoff_24h:
|
||||
n_24h += 1
|
||||
fa = art.get("fetched_at")
|
||||
if fa:
|
||||
try:
|
||||
if datetime.fromisoformat(fa) >= cutoff_24h:
|
||||
n_24h += 1
|
||||
except ValueError:
|
||||
n_24h += 1 # 时间格式异常时保守计入
|
||||
else:
|
||||
n_24h += 1 # 时间缺失时保守计入
|
||||
raw_by_source_24h[name] = n_24h
|
||||
raw_total_24h += n_24h
|
||||
|
||||
|
||||
Reference in New Issue
Block a user