feat: 日报事件多新闻源入库(news_event.sources 列)
- ExtractedEvent 增加 sources 字段(主源居首去重,旧产物兜底 [source_id]) - extract_event(_async) 增加 sources 参数;run_event_extraction 读取 deduped uniques 的 sources 透传 - reporter _load_events_from_dir 读取 sources → EventRow.sources - news_event 建表加 sources TEXT 列;init_schema 幂等迁移旧表(捕获 Duplicate column);save_report INSERT 写 JSON 多源 - 新增测试:ExtractedEvent/EventRow.sources 兜底与保序、_load_events_from_dir 多源
This commit is contained in:
+14
-4
@@ -11,7 +11,7 @@ import pymysql
|
||||
from loguru import logger
|
||||
|
||||
from .models import ReportData
|
||||
from .schema import DDL_STATEMENTS
|
||||
from .schema import _ALTER_ADD_SOURCES, DDL_STATEMENTS
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -60,10 +60,19 @@ def connect(cfg: DbConfig | None = None) -> pymysql.Connection:
|
||||
|
||||
|
||||
def init_schema(conn: pymysql.Connection) -> None:
|
||||
"""建表(CREATE TABLE IF NOT EXISTS ×2),幂等。"""
|
||||
"""建表(CREATE TABLE IF NOT EXISTS ×2)+ 旧表 sources 列迁移,幂等。"""
|
||||
with conn.cursor() as cur:
|
||||
for ddl in DDL_STATEMENTS:
|
||||
cur.execute(ddl)
|
||||
# 旧库兼容:news_event 已存在但缺 sources 列时补列
|
||||
try:
|
||||
cur.execute(_ALTER_ADD_SOURCES)
|
||||
logger.info("news_event 迁移:新增 sources 多源列")
|
||||
except pymysql.err.OperationalError as e:
|
||||
if e.args and "Duplicate column" in str(e.args[0]):
|
||||
logger.debug("news_event.sources 列已存在,跳过迁移")
|
||||
else:
|
||||
raise
|
||||
conn.commit()
|
||||
logger.info("news_report / news_event 建表完成")
|
||||
|
||||
@@ -112,8 +121,8 @@ def save_report(conn: pymysql.Connection, report: ReportData) -> int:
|
||||
"""
|
||||
INSERT INTO news_event
|
||||
(report_id, section, rank, importance, event_type, title,
|
||||
summary, sentiment, source, url)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
summary, sentiment, source, sources, url)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(
|
||||
report_id,
|
||||
@@ -125,6 +134,7 @@ def save_report(conn: pymysql.Connection, report: ReportData) -> int:
|
||||
ev.summary,
|
||||
ev.sentiment,
|
||||
ev.source,
|
||||
json.dumps(ev.sources, ensure_ascii=False) if ev.sources else None,
|
||||
ev.url,
|
||||
),
|
||||
)
|
||||
|
||||
+2
-1
@@ -18,7 +18,8 @@ class EventRow(BaseModel):
|
||||
title: str
|
||||
summary: str | None = None
|
||||
sentiment: str | None = None # positive | negative | neutral | ''
|
||||
source: str | None = None
|
||||
source: str | None = None # 主源
|
||||
sources: list[str] | None = None # 全部来源(主源居首),多源新闻记录
|
||||
url: str | None = None
|
||||
|
||||
|
||||
|
||||
+8
-1
@@ -33,7 +33,8 @@ DDL_STATEMENTS: list[str] = [
|
||||
title VARCHAR(512) NOT NULL COMMENT '标题',
|
||||
summary TEXT NULL COMMENT '摘要/正文',
|
||||
sentiment VARCHAR(8) NULL COMMENT 'positive/negative/neutral',
|
||||
source VARCHAR(64) NULL COMMENT '来源',
|
||||
source VARCHAR(64) NULL COMMENT '主来源',
|
||||
sources TEXT NULL COMMENT '全部来源 JSON 数组(主源居首),多源新闻记录',
|
||||
url VARCHAR(512) NULL COMMENT '原文链接',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
KEY idx_report_section (report_id, section),
|
||||
@@ -41,3 +42,9 @@ DDL_STATEMENTS: list[str] = [
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='日报事件明细'
|
||||
""",
|
||||
]
|
||||
|
||||
# 旧库迁移:为已存在的 news_event 表补充 sources 列(MySQL 无 ADD COLUMN IF NOT EXISTS)
|
||||
_ALTER_ADD_SOURCES = (
|
||||
"ALTER TABLE news_event ADD COLUMN sources TEXT NULL "
|
||||
"COMMENT '全部来源 JSON 数组(主源居首),多源新闻记录' AFTER source"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user