初始化
This commit is contained in:
@@ -0,0 +1,258 @@
|
||||
"""英文正文提取引擎
|
||||
|
||||
策略：
|
||||
1. 优先 trafilatura 从原始 HTML 提取（除导航/广告/页脚）
|
||||
2. 回退 Crawl4AI Markdown（如果 HTML 不可用）
|
||||
3. 元数据提取（发布时间、作者）
|
||||
所有业务参数从 configs/system.yaml 的 extractor 节读取。
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import trafilatura
|
||||
import yaml
|
||||
from htmldate import find_date
|
||||
|
||||
from extractor.models import ProcessedArticle
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _load_extractor_config() -> dict:
|
||||
"""从 system.yaml 读取 extractor 配置节"""
|
||||
config_path = Path("configs/system.yaml")
|
||||
if config_path.exists():
|
||||
try:
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
raw = yaml.safe_load(f)
|
||||
return raw.get("extractor", {})
|
||||
except Exception:
|
||||
logger.warning("读取 system.yaml 失败，使用默认值")
|
||||
return {}
|
||||
|
||||
_cfg = _load_extractor_config()
|
||||
|
||||
# 最小正文字数阈值
|
||||
MIN_CONTENT_WORDS = int(_cfg.get("min_content_words", 50))
|
||||
|
||||
|
||||
def extract_article(
|
||||
source_id: str,
|
||||
source_name: str,
|
||||
url: str,
|
||||
url_hash: str,
|
||||
html_path: str,
|
||||
md_path: str,
|
||||
crawl_publish_time: str = "",
|
||||
) -> ProcessedArticle:
|
||||
"""提取单篇文章的正文
|
||||
|
||||
Args:
|
||||
source_id: 新闻源 ID
|
||||
source_name: 新闻源名称
|
||||
url: 文章 URL
|
||||
url_hash: URL SHA256 前 16 位
|
||||
html_path: 原始 HTML 文件路径
|
||||
md_path: Crawl4AI 生成的 Markdown 文件路径
|
||||
crawl_publish_time: RSS/Crawler 提取的发布时间（回退用）
|
||||
|
||||
Returns:
|
||||
ProcessedArticle
|
||||
"""
|
||||
article = ProcessedArticle(
|
||||
source_id=source_id,
|
||||
source_name=source_name,
|
||||
url=url,
|
||||
url_hash=url_hash,
|
||||
html_path=html_path,
|
||||
md_path=md_path,
|
||||
)
|
||||
|
||||
html_p = Path(html_path) if html_path else None
|
||||
md_p = Path(md_path) if md_path else None
|
||||
|
||||
# ── 策略 1: trafilatura 从 HTML 提取 ──────────────
|
||||
if html_p and html_p.exists():
|
||||
try:
|
||||
html_raw = html_p.read_text(encoding="utf-8")
|
||||
content = trafilatura.extract(
|
||||
html_raw,
|
||||
include_comments=False,
|
||||
include_tables=False,
|
||||
no_fallback=False,
|
||||
favor_precision=True,
|
||||
)
|
||||
if content and _count_words(content) >= MIN_CONTENT_WORDS:
|
||||
article.content = content.strip()
|
||||
article.extractor = "trafilatura"
|
||||
article.word_count = _count_words(article.content)
|
||||
|
||||
# 元数据
|
||||
article.title = _extract_title(html_raw) or ""
|
||||
article.publish_time = (
|
||||
_extract_publish_time(html_raw, url=url)
|
||||
or crawl_publish_time
|
||||
or ""
|
||||
)
|
||||
article.author = _extract_author(html_raw) or ""
|
||||
|
||||
article.status = "success"
|
||||
return article
|
||||
except Exception as e:
|
||||
logger.warning("[%s] trafilatura 提取失败: %s — %s", source_id, url_hash, e)
|
||||
|
||||
# ── 策略 2: Crawl4AI Markdown 回退 ─────────────────
|
||||
if md_p and md_p.exists():
|
||||
try:
|
||||
md_content = md_p.read_text(encoding="utf-8")
|
||||
if _count_words(md_content) >= MIN_CONTENT_WORDS:
|
||||
article.content = _clean_markdown(md_content)
|
||||
article.extractor = "crawl4ai_md"
|
||||
article.word_count = _count_words(article.content)
|
||||
|
||||
# 从 Markdown 中提取元数据（RSS 源会写入）
|
||||
article.title = _extract_from_md(
|
||||
md_content, r"^#\s+(.+)$", article.title
|
||||
)
|
||||
article.publish_time = (
|
||||
_extract_from_md(
|
||||
md_content, r"\*\*发布时间\*\*:?\s*([^\n]+)", ""
|
||||
)
|
||||
or _extract_publish_time("", url=url)
|
||||
or crawl_publish_time
|
||||
or ""
|
||||
)
|
||||
|
||||
article.status = "success"
|
||||
return article
|
||||
except Exception as e:
|
||||
logger.warning("[%s] Markdown 回退失败: %s — %s", source_id, url_hash, e)
|
||||
|
||||
# ── 失败 ─────────────────────────────────────────
|
||||
article.status = "no_content"
|
||||
article.error = "No extractable content"
|
||||
return article
|
||||
|
||||
|
||||
# ── 辅助函数 ────────────────────────────────────────
|
||||
|
||||
|
||||
def _count_words(text: str) -> int:
|
||||
"""英文词数统计"""
|
||||
return len(text.split()) if text else 0
|
||||
|
||||
|
||||
def _extract_title(html: str) -> str:
|
||||
"""从 HTML <title> 标签提取标题"""
|
||||
import re
|
||||
m = re.search(r"<title[^>]*>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
|
||||
if m:
|
||||
title = re.sub(r"<[^>]+>", "", m.group(1))
|
||||
return title.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_publish_time(html: str, url: str = "") -> str:
|
||||
"""从 HTML 提取发布时间（含回退策略）。
|
||||
|
||||
优先级:
|
||||
1. htmldate 提取（元数据/标签/正文）
|
||||
2. <meta property="article:published_time"> 等常见标签
|
||||
3. URL 中的日期（/YYYY/MM/DD/ 或 /YYYYMMDD/）
|
||||
"""
|
||||
# 策略 1: htmldate
|
||||
try:
|
||||
result = find_date(html)
|
||||
if result:
|
||||
return result
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 策略 2: 常见 meta 标签
|
||||
if html:
|
||||
import re
|
||||
meta_patterns = [
|
||||
r'<meta[^>]+property=["\']article:published_time["\'][^>]+content=["\']([^"\']+)',
|
||||
r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']article:published_time',
|
||||
r'<meta[^>]+name=["\']pubdate["\'][^>]+content=["\']([^"\']+)',
|
||||
r'<meta[^>]+name=["\']publish_date["\'][^>]+content=["\']([^"\']+)',
|
||||
r'<meta[^>]+name=["\']date["\'][^>]+content=["\']([^"\']+)',
|
||||
r'<time[^>]+datetime=["\']([^"\']+)',
|
||||
]
|
||||
for pat in meta_patterns:
|
||||
m = re.search(pat, html, re.IGNORECASE)
|
||||
if m:
|
||||
date_str = m.group(1).strip()
|
||||
if date_str:
|
||||
return date_str
|
||||
|
||||
# 策略 3: URL 中的日期
|
||||
if url:
|
||||
from datetime import datetime as dt
|
||||
import re
|
||||
url_patterns = [
|
||||
r"/(\d{4})/(\d{2})/(\d{2})/",
|
||||
r"/(\d{4})(\d{2})(\d{2})/",
|
||||
r"-(\d{4})(\d{2})(\d{2})(?:[/-]|$)",
|
||||
r"-(\d{4})-(\d{2})-(\d{2})[/-]",
|
||||
]
|
||||
for pat in url_patterns:
|
||||
m = re.search(pat, url)
|
||||
if m:
|
||||
try:
|
||||
y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
return dt(y, mo, d).isoformat()
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_author(html: str) -> str:
|
||||
"""从 HTML meta 标签提取作者"""
|
||||
import re
|
||||
for pat in [
|
||||
r'<meta[^>]+name=["\']author["\'][^>]+content=["\']([^"\']+)',
|
||||
r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+name=["\']author',
|
||||
]:
|
||||
m = re.search(pat, html, re.IGNORECASE)
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_from_md(md: str, pattern: str, default: str) -> str:
|
||||
"""从 Markdown 内容中按正则提取第一个捕获组，失败返回 default。"""
|
||||
import re
|
||||
m = re.search(pattern, md)
|
||||
return m.group(1).strip() if m else default
|
||||
|
||||
|
||||
def _clean_markdown(md: str) -> str:
|
||||
"""清理 Crawl4AI MD 中的导航/广告噪音
|
||||
|
||||
规则：移除明显非正文行（短链接行、纯导航行等）
|
||||
"""
|
||||
lines = md.split("\n")
|
||||
cleaned: list[str] = []
|
||||
skip_patterns = [
|
||||
"ADVERTISEMENT",
|
||||
"Continue Reading Below",
|
||||
"Sign in",
|
||||
"Log in",
|
||||
"Subscribe",
|
||||
"Advertisement",
|
||||
]
|
||||
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
# 跳过明显广告/导航行
|
||||
if any(p in stripped for p in skip_patterns):
|
||||
continue
|
||||
# 跳过纯导航链接行（[text](url) 且整行只有链接）
|
||||
if stripped.startswith("[") and stripped.endswith(")") and stripped.count("[") <= 3:
|
||||
continue
|
||||
cleaned.append(line)
|
||||
|
||||
return "\n".join(cleaned)
|
||||
@@ -0,0 +1,22 @@
|
||||
"""正文提取数据模型"""
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ProcessedArticle(BaseModel):
|
||||
"""经过正文提取清洗后的文章"""
|
||||
|
||||
source_id: str
|
||||
source_name: str
|
||||
url: str
|
||||
url_hash: str
|
||||
title: str = ""
|
||||
content: str = "" # 清洗后的正文（英文）
|
||||
publish_time: str = "" # ISO 8601
|
||||
author: str = ""
|
||||
word_count: int = 0
|
||||
md_path: str = "" # 原始 Markdown 路径
|
||||
html_path: str = "" # 原始 HTML 路径
|
||||
status: str = "success" # success | no_content | failed
|
||||
extractor: str = "trafilatura" # trafilatura | crawl4ai_md | none
|
||||
error: str = ""
|
||||
@@ -0,0 +1,175 @@
|
||||
"""正文提取管道：批量处理 raw 目录下的文章"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from crawler.storage import load_index
|
||||
from crawler.utils import get_news_day
|
||||
from extractor.extractor import extract_article
|
||||
from extractor.models import ProcessedArticle
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_raw_data_sources(base_dir: str = "data/raw") -> list[str]:
|
||||
"""扫描 data/raw/ 下所有源 ID
|
||||
|
||||
Args:
|
||||
base_dir: raw 数据根目录
|
||||
|
||||
Returns:
|
||||
源 ID 列表
|
||||
"""
|
||||
raw_path = Path(base_dir)
|
||||
if not raw_path.exists():
|
||||
return []
|
||||
return sorted([
|
||||
d.name for d in raw_path.iterdir()
|
||||
if d.is_dir() and not d.name.startswith(".")
|
||||
])
|
||||
|
||||
|
||||
def process_source(
|
||||
source_id: str,
|
||||
date_str: str | None = None,
|
||||
) -> list[ProcessedArticle]:
|
||||
"""处理单个源的文章正文提取
|
||||
|
||||
Args:
|
||||
source_id: 新闻源 ID
|
||||
date_str: 日期 YYYYMMDD，默认当前新闻日
|
||||
|
||||
Returns:
|
||||
ProcessedArticle 列表
|
||||
"""
|
||||
if date_str is None:
|
||||
date_str = get_news_day()
|
||||
|
||||
logger.info("━━━ 正文提取 [%s] %s ━━━", source_id, date_str)
|
||||
|
||||
# 读取 raw index
|
||||
articles = load_index(source_id, date_str)
|
||||
if not articles:
|
||||
logger.warning("[%s] %s 无待处理文章", source_id, date_str)
|
||||
return []
|
||||
|
||||
# 输出目录
|
||||
out_dir = Path(f"data/processed/{source_id}/{date_str}")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
results: list[ProcessedArticle] = []
|
||||
success = 0
|
||||
no_content = 0
|
||||
failed = 0
|
||||
|
||||
# 增量：跳过已处理的文章
|
||||
skipped = 0
|
||||
for article in articles[:]:
|
||||
out_file = out_dir / f"{article.url_hash}.json"
|
||||
if out_file.exists():
|
||||
articles.remove(article)
|
||||
skipped += 1
|
||||
if skipped > 0:
|
||||
logger.info("[%s] 增量跳过 %d 篇已处理，剩余 %d 篇",
|
||||
source_id, skipped, len(articles))
|
||||
|
||||
for article in articles:
|
||||
# 确保 html_path / md_path 是绝对路径（相对于 data/raw/）
|
||||
html_path = str(Path("data/raw") / source_id / date_str / f"{article.url_hash}.html")
|
||||
md_path = str(Path("data/raw") / source_id / date_str / f"{article.url_hash}.md")
|
||||
|
||||
# 如果 index 中已有路径，优先使用
|
||||
if article.html_path:
|
||||
html_path = article.html_path
|
||||
if article.md_path:
|
||||
md_path = article.md_path
|
||||
|
||||
processed = extract_article(
|
||||
source_id=source_id,
|
||||
source_name=article.source_name,
|
||||
url=article.url,
|
||||
url_hash=article.url_hash,
|
||||
html_path=html_path,
|
||||
md_path=md_path,
|
||||
crawl_publish_time=article.publish_time,
|
||||
)
|
||||
|
||||
# 写入 JSON 输出
|
||||
out_file = out_dir / f"{article.url_hash}.json"
|
||||
out_file.write_text(
|
||||
processed.model_dump_json(indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
results.append(processed)
|
||||
|
||||
if processed.status == "success":
|
||||
success += 1
|
||||
logger.debug("[%s] ✅ %s (%d words, %s)",
|
||||
source_id, processed.title[:40], processed.word_count, processed.extractor)
|
||||
elif processed.status == "no_content":
|
||||
no_content += 1
|
||||
logger.warning("[%s] ⚠️ 无正文: %s", source_id, processed.url[:80])
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
logger.info("[%s] 完成: 成功 %d / 无内容 %d / 失败 %d",
|
||||
source_id, success, no_content, failed)
|
||||
|
||||
# 写入处理索引
|
||||
index_path = out_dir / "index.jsonl"
|
||||
with open(index_path, "a", encoding="utf-8") as f:
|
||||
for r in results:
|
||||
if r.status == "success":
|
||||
f.write(json.dumps({
|
||||
"url_hash": r.url_hash,
|
||||
"title": r.title,
|
||||
"word_count": r.word_count,
|
||||
"extractor": r.extractor,
|
||||
"publish_time": r.publish_time,
|
||||
}, ensure_ascii=False) + "\n")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def process_all_sources(
|
||||
source_filter: str | None = None,
|
||||
date_str: str | None = None,
|
||||
) -> dict:
|
||||
"""处理所有源的文章正文提取
|
||||
|
||||
Args:
|
||||
source_filter: 可选，只处理指定源
|
||||
date_str: 日期，默认当前新闻日
|
||||
|
||||
Returns:
|
||||
统计 dict
|
||||
"""
|
||||
if date_str is None:
|
||||
date_str = get_news_day()
|
||||
|
||||
start_time = datetime.now()
|
||||
|
||||
if source_filter:
|
||||
sources = [source_filter] if source_filter in get_raw_data_sources() else []
|
||||
else:
|
||||
sources = get_raw_data_sources()
|
||||
|
||||
logger.info("══════ 开始正文提取 %d 个源，日期: %s ══════", len(sources), date_str)
|
||||
|
||||
total = 0
|
||||
for src in sources:
|
||||
results = process_source(src, date_str)
|
||||
total += len([r for r in results if r.status == "success"])
|
||||
|
||||
elapsed = (datetime.now() - start_time).total_seconds()
|
||||
logger.info("══════ 提取完成: %d 篇文章，耗时 %.1f 秒 ══════", total, elapsed)
|
||||
|
||||
return {
|
||||
"sources_processed": len(sources),
|
||||
"total_articles": total,
|
||||
"elapsed_sec": elapsed,
|
||||
"date": date_str,
|
||||
}
|
||||
Reference in New Issue
Block a user