simon
7ea8925209
- run_xwlb: 新增 --date(处理日)参数,落盘改为处理日目录,下游 M2-M6 打通
- run_xwlb: 幂等落盘(按 url_hash 去重,重写 index 替代裸追加)
- pipeline: xwlb 步骤传 --date {date}
- run_extractor: xwlb publish_time 从 url 兜底解析真实播出日
- 新增 tests/test_run_xwlb.py 9 个测试
- 清理 60 天历史死数据(用户决策 A1)
- docs: 文档重构收尾(删除 deployment.md,已在服务器直接修改)
413 lines
14 KiB
Python
413 lines
14 KiB
Python
"""M2 批量正文提取入口脚本。
|
|
|
|
输入: data/raw/{source}/{YYYYMMDD}/index.jsonl (M1 产物)
|
|
输出: data/processed/{source}/{YYYYMMDD}/{url_hash}.json (Article)
|
|
data/processed/{source}/{YYYYMMDD}/index.jsonl (扁平元数据,便于检索)
|
|
|
|
用法:
|
|
uv run python -m scripts.run_extractor # 处理今日所有源
|
|
uv run python -m scripts.run_extractor --date 20260616
|
|
uv run python -m scripts.run_extractor --source sina --date 20260616
|
|
uv run python -m scripts.run_extractor --raw-root data/raw --out-root data/processed
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import json
|
|
import sys
|
|
from datetime import date, datetime
|
|
from pathlib import Path
|
|
|
|
from loguru import logger
|
|
from pydantic import ValidationError
|
|
|
|
from extractor import Article, ExtractError, extract_article
|
|
from extractor.parser import _url_hash
|
|
|
|
|
|
def _setup_logger(level: str) -> None:
|
|
logger.remove()
|
|
logger.add(
|
|
sys.stderr,
|
|
level=level,
|
|
format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
|
|
)
|
|
log_path = Path("logs") / "extractor.log"
|
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
|
|
|
|
|
|
def _iter_article_records(raw_dir: Path) -> list[dict]:
|
|
"""读取 M1 产物的 index.jsonl,返回所有可处理的记录。
|
|
|
|
新闻源: stage=article + success + html_file
|
|
cninfo: 有 json_file 字段(CninfoItem 格式)
|
|
"""
|
|
index_path = raw_dir / "index.jsonl"
|
|
if not index_path.is_file():
|
|
return []
|
|
out: list[dict] = []
|
|
with index_path.open("r", encoding="utf-8") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
rec = json.loads(line)
|
|
except json.JSONDecodeError as e:
|
|
logger.warning("跳过非法 jsonl 行 in {}: {}", index_path, e)
|
|
continue
|
|
# cninfo: 新格式(CninfoItem → json_file)
|
|
if rec.get("source_id") == "cninfo" or rec.get("json_file"):
|
|
if rec.get("json_file"):
|
|
out.append(rec)
|
|
continue
|
|
# 新闻源: 旧格式(CrawlResult → html_file)
|
|
if rec.get("stage") == "article" and rec.get("success") and rec.get("html_file"):
|
|
out.append(rec)
|
|
return out
|
|
|
|
|
|
def _process_one(rec: dict, raw_dir: Path, out_dir: Path,
|
|
body_xpath_map: dict[str, str] | None = None) -> Article | None:
|
|
"""处理单条记录,失败返回 None。cninfo 等结构化源跳过 GNE。"""
|
|
src_id = rec.get("source_id", "")
|
|
|
|
# cninfo: 新格式(json_file)或旧格式(html_file+meta JSON)直接解析
|
|
json_file = rec.get("json_file", "")
|
|
html_file = rec.get("html_file", "")
|
|
is_cninfo = (src_id == "cninfo" or bool(json_file))
|
|
|
|
if is_cninfo:
|
|
if json_file:
|
|
json_path = raw_dir / json_file
|
|
if json_path.is_file():
|
|
return _process_cninfo_v2(rec, json_path, out_dir)
|
|
else:
|
|
logger.warning("cninfo JSON 文件丢失: {}", json_path)
|
|
return None
|
|
# 旧格式兼容: html_file + <meta>JSON
|
|
if html_file:
|
|
html_path = raw_dir / html_file
|
|
if html_path.is_file():
|
|
return _process_cninfo(rec, html_path, out_dir)
|
|
return None
|
|
|
|
html_path = raw_dir / html_file
|
|
if not html_path or not html_path.is_file():
|
|
logger.warning("HTML 文件丢失: {}", html_path)
|
|
return None
|
|
|
|
html = html_path.read_text(encoding="utf-8", errors="ignore")
|
|
extra_config: dict[str, str] = {}
|
|
if body_xpath_map and src_id in body_xpath_map:
|
|
extra_config["body_xpath"] = body_xpath_map[src_id]
|
|
try:
|
|
article = extract_article(
|
|
html=html,
|
|
source_id=src_id,
|
|
url=rec["url"],
|
|
extra_config=extra_config or None,
|
|
)
|
|
except ExtractError as e:
|
|
logger.warning("提取失败 {} {}: {}", rec["source_id"], rec["url"], e.reason)
|
|
return None
|
|
|
|
# xwlb 的假 HTML 无时间节点,从 url(xwlb://YYYY-MM-DD/sid)兜底解析真实播出日
|
|
if src_id == "xwlb" and article.publish_time is None:
|
|
article = _fill_xwlb_publish_time(article)
|
|
|
|
return _save_article(article, out_dir)
|
|
|
|
|
|
def _fill_xwlb_publish_time(article: Article) -> Article:
|
|
"""从 xwlb url 解析播出日期填充 publish_time,失败时原样返回。"""
|
|
import re as _re
|
|
|
|
m = _re.match(r"xwlb://(\d{4}-\d{2}-\d{2})", article.url)
|
|
if not m:
|
|
return article
|
|
try:
|
|
pt = datetime.strptime(m.group(1), "%Y-%m-%d")
|
|
except ValueError:
|
|
return article
|
|
return article.model_copy(update={
|
|
"publish_time": pt,
|
|
"publish_time_raw": m.group(1),
|
|
})
|
|
|
|
|
|
def _process_cninfo(rec: dict, html_path: Path, out_dir: Path) -> Article | None:
|
|
"""处理 cninfo 公告记录:从 meta JSON 解析结构化数据。"""
|
|
import re
|
|
html = html_path.read_text(encoding="utf-8", errors="ignore")
|
|
# 提取 <meta>{...}</meta> 中的 JSON
|
|
m = re.search(r"<meta>(.+?)</meta>", html, re.DOTALL)
|
|
if not m:
|
|
logger.warning("cninfo HTML 不含 meta JSON: {}", html_path)
|
|
return None
|
|
try:
|
|
meta = json.loads(m.group(1))
|
|
except json.JSONDecodeError:
|
|
logger.warning("cninfo meta JSON 解析失败: {}", html_path)
|
|
return None
|
|
|
|
sec_code = (meta.get("secCode") or "").strip()
|
|
sec_name = (meta.get("secName") or "").strip()
|
|
ann_type = (meta.get("announcementType") or "").strip()
|
|
pdf_url = (meta.get("pdfUrl") or "").strip()
|
|
title = rec.get("title") or meta.get("title") or ""
|
|
|
|
# 构建正文:结构化摘要 + 公告类别翻译
|
|
content_parts = [f"公司: {sec_name}({sec_code})", f"公告标题: {title}"]
|
|
if ann_type:
|
|
content_parts.append(f"公告类别编码: {ann_type}")
|
|
if pdf_url:
|
|
content_parts.append(f"PDF: {pdf_url}")
|
|
content = "\n".join(content_parts)
|
|
|
|
# 时间
|
|
fetched = rec.get("fetched_at")
|
|
publish_time = None
|
|
if isinstance(fetched, str):
|
|
try:
|
|
from datetime import datetime as dt
|
|
publish_time = dt.fromisoformat(fetched)
|
|
except ValueError:
|
|
pass
|
|
|
|
article = Article(
|
|
source_id="cninfo",
|
|
url=rec.get("url") or "",
|
|
url_hash=_url_hash(rec.get("url") or ""),
|
|
title=title,
|
|
content=content,
|
|
author=sec_name,
|
|
source_name="巨潮资讯网",
|
|
publish_time=publish_time,
|
|
publish_time_raw=fetched,
|
|
word_count=len(content),
|
|
)
|
|
return _save_article(article, out_dir)
|
|
|
|
|
|
def _process_cninfo_v2(rec: dict, json_path: Path, out_dir: Path) -> Article | None:
|
|
"""处理 cninfo v2 格式: 直接读取 CninfoItem JSON 并转换为 Article。"""
|
|
try:
|
|
item_data = json.loads(json_path.read_text(encoding="utf-8"))
|
|
except (json.JSONDecodeError, OSError) as e:
|
|
logger.warning("cninfo JSON 读取失败 {}: {}", json_path, e)
|
|
return None
|
|
|
|
stock_code = item_data.get("stock_code", "")
|
|
stock_name = item_data.get("stock_name", "")
|
|
title = item_data.get("title", "")
|
|
content = item_data.get("content", "")
|
|
publish_time_str = item_data.get("publish_time", "")
|
|
item_type = item_data.get("item_type", "announcement")
|
|
url = item_data.get("url", "")
|
|
extra = item_data.get("extra", {})
|
|
|
|
# 类型中文映射
|
|
type_map = {
|
|
"announcement": "公告",
|
|
"research": "投资者调研",
|
|
"irm": "互动问答",
|
|
}
|
|
type_cn = type_map.get(item_type, item_type)
|
|
|
|
# 构建正文
|
|
content_parts = [
|
|
f"公司: {stock_name}({stock_code})",
|
|
f"类型: {type_cn}",
|
|
f"标题: {title}",
|
|
]
|
|
if extra.get("announcement_type"):
|
|
content_parts.append(f"公告类别: {extra['announcement_type']}")
|
|
if url:
|
|
content_parts.append(f"原文链接: {url}")
|
|
if content:
|
|
content_parts.append(f"\n正文:\n{content}")
|
|
full_content = "\n".join(content_parts)
|
|
|
|
# 发布时间解析
|
|
publish_time = None
|
|
publish_time_raw = publish_time_str
|
|
if publish_time_str:
|
|
try:
|
|
from datetime import datetime as dt
|
|
publish_time = dt.strptime(publish_time_str, "%Y-%m-%d")
|
|
except ValueError:
|
|
try:
|
|
publish_time = dt.fromisoformat(publish_time_str)
|
|
except ValueError:
|
|
pass
|
|
|
|
article = Article(
|
|
source_id="cninfo",
|
|
url=url,
|
|
url_hash=_url_hash(url or title),
|
|
title=title,
|
|
content=full_content,
|
|
author=stock_name,
|
|
source_name="巨潮资讯网",
|
|
publish_time=publish_time,
|
|
publish_time_raw=publish_time_raw,
|
|
word_count=len(full_content),
|
|
item_type=item_type,
|
|
)
|
|
return _save_article(article, out_dir)
|
|
|
|
|
|
def _append_index(article: Article, out_dir: Path) -> None:
|
|
"""把 Article 的扁平摘要追加到 index.jsonl。"""
|
|
flat = article.model_dump(exclude={"content", "images"}, mode="json")
|
|
flat["article_file"] = f"{article.url_hash}.json"
|
|
flat["content_preview"] = article.content[:80]
|
|
with (out_dir / "index.jsonl").open("a", encoding="utf-8") as f:
|
|
f.write(json.dumps(flat, ensure_ascii=False) + "\n")
|
|
|
|
|
|
def _save_article(article: Article, out_dir: Path) -> Article:
|
|
"""保存 Article JSON 并追加 index。"""
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
article_path = out_dir / f"{article.url_hash}.json"
|
|
article_path.write_text(article.model_dump_json(indent=2), encoding="utf-8")
|
|
_append_index(article, out_dir)
|
|
return article
|
|
|
|
|
|
def _process_source_day(
|
|
source_id: str,
|
|
day: str,
|
|
raw_root: Path,
|
|
out_root: Path,
|
|
body_xpath_map: dict[str, str] | None = None,
|
|
*,
|
|
force: bool = False,
|
|
) -> tuple[int, int, int]:
|
|
"""处理单个源单日。返回 (成功数, 总数, 跳过数)。
|
|
|
|
默认增量:已提取的文章(输出目录已有 {url_hash}.json)跳过提取,仅回补 index 行;
|
|
force=True 时全量重提取并重建 index。
|
|
"""
|
|
raw_dir = raw_root / source_id / day
|
|
out_dir = out_root / source_id / day
|
|
|
|
records = _iter_article_records(raw_dir)
|
|
if not records:
|
|
logger.info("源 {} 日期 {} 无可处理记录", source_id, day)
|
|
return 0, 0, 0
|
|
|
|
# 全量模式:重建 index;增量模式:保留旧 index 追加新条目
|
|
old_index = out_dir / "index.jsonl"
|
|
if force and old_index.exists():
|
|
old_index.unlink()
|
|
|
|
succ = 0
|
|
skipped = 0
|
|
for rec in records:
|
|
url_hash = rec.get("url_hash") or _url_hash(rec.get("url") or "")
|
|
existing = out_dir / f"{url_hash}.json"
|
|
if not force and existing.is_file():
|
|
# 增量:跳过已提取,回补 index 行保持摘要完整
|
|
skipped += 1
|
|
with contextlib.suppress(json.JSONDecodeError, ValidationError, OSError):
|
|
_append_index(
|
|
Article.model_validate(json.loads(existing.read_text(encoding="utf-8"))),
|
|
out_dir,
|
|
)
|
|
continue
|
|
article = _process_one(rec, raw_dir, out_dir, body_xpath_map)
|
|
if article is not None:
|
|
succ += 1
|
|
total = len(records)
|
|
# 增量模式下「跳过已提取」视为已成功处理,避免全跳过时误报成功率 0%
|
|
rate = (succ + skipped) / max(total, 1)
|
|
logger.info(
|
|
"源 {} 日期 {} 提取完成: {}/{} 成功率 {:.0%} (跳过已提取 {})",
|
|
source_id,
|
|
day,
|
|
succ,
|
|
total,
|
|
rate,
|
|
skipped,
|
|
)
|
|
return succ, total, skipped
|
|
|
|
|
|
def _list_source_dirs(raw_root: Path) -> list[str]:
|
|
"""列出 raw_root 下所有源 id(子目录名)。"""
|
|
if not raw_root.is_dir():
|
|
return []
|
|
return sorted(p.name for p in raw_root.iterdir() if p.is_dir())
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="A 股新闻正文提取 (M2)")
|
|
parser.add_argument("--raw-root", default="data/raw", help="M1 抓取产物根目录")
|
|
parser.add_argument("--out-root", default="data/processed", help="M2 提取结果根目录")
|
|
parser.add_argument("--source", default=None, help="只处理单个源 id,默认全部")
|
|
parser.add_argument(
|
|
"--date",
|
|
default=date.today().strftime("%Y%m%d"),
|
|
help="处理日期 YYYYMMDD,默认今日",
|
|
)
|
|
parser.add_argument("--force", action="store_true",
|
|
help="强制全量重提取(默认跳过已提取文章)")
|
|
parser.add_argument("--log-level", default="INFO")
|
|
args = parser.parse_args()
|
|
|
|
_setup_logger(args.log_level)
|
|
raw_root = Path(args.raw_root)
|
|
out_root = Path(args.out_root)
|
|
|
|
sources = [args.source] if args.source else _list_source_dirs(raw_root)
|
|
if not sources:
|
|
logger.error("{} 下未发现任何源目录", raw_root)
|
|
return 2
|
|
|
|
# 加载源配置,构建 source_id → body_xpath 映射
|
|
body_xpath_map: dict[str, str] = {}
|
|
try:
|
|
from crawler.config import load_crawler_config
|
|
cfg = load_crawler_config()
|
|
for s in cfg.sources:
|
|
if s.body_xpath:
|
|
body_xpath_map[s.id] = s.body_xpath
|
|
if body_xpath_map:
|
|
logger.info("已加载 body_xpath 配置: {}", dict(body_xpath_map))
|
|
except Exception:
|
|
pass
|
|
|
|
started = datetime.now()
|
|
total_succ = 0
|
|
total_all = 0
|
|
total_skipped = 0
|
|
for src in sources:
|
|
succ, total, skipped = _process_source_day(
|
|
src, args.date, raw_root, out_root, body_xpath_map, force=args.force
|
|
)
|
|
total_succ += succ
|
|
total_all += total
|
|
total_skipped += skipped
|
|
|
|
elapsed = (datetime.now() - started).total_seconds()
|
|
# 增量模式下跳过已提取视为成功
|
|
rate = (total_succ + total_skipped) / max(total_all, 1)
|
|
logger.info(
|
|
"全部完成: {}/{} 成功率 {:.0%} (跳过已提取 {}) 用时 {:.1f}s",
|
|
total_succ,
|
|
total_all,
|
|
rate,
|
|
total_skipped,
|
|
elapsed,
|
|
)
|
|
return 0 if rate >= 0.9 or total_all == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|