初始化
This commit is contained in:
@@ -0,0 +1,393 @@
|
||||
"""RSS/Atom Feed 抓取模块。
|
||||
|
||||
用于有 RSS 服务的新闻源（如 MarketWatch），绕过网页反爬。
|
||||
支持 RSS 2.0、Atom、Google News RSS 三种格式。
|
||||
|
||||
Google News RSS: 处理标题去来源后缀、链接为 Google 跳转 URL。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from html import unescape
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
from xml.etree import ElementTree
|
||||
|
||||
import httpx
|
||||
|
||||
from crawler.models import ArticleItem, CrawlResult, SourceConfig
|
||||
from crawler.utils import get_news_day
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── HTML 标签清洗 ──────────────────────────────────────
|
||||
|
||||
|
||||
def _strip_html(text: str) -> str:
|
||||
"""去除 HTML 标签，保留纯文本。"""
|
||||
if not text:
|
||||
return ""
|
||||
text = unescape(text)
|
||||
return re.sub(r"<[^>]+>", "", text).strip()
|
||||
|
||||
|
||||
# ── Namespace ──────────────────────────────────────────
|
||||
|
||||
|
||||
def _ns(tag: str) -> str:
|
||||
"""Atom namespace helper。"""
|
||||
return f"{{http://www.w3.org/2005/Atom}}{tag}"
|
||||
|
||||
|
||||
def _compute_url_hash(url: str) -> str:
|
||||
return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
# 非内容类扩展名 — RSS 项也可能包含静态资源
|
||||
_SKIP_EXT = re.compile(
|
||||
r"\.(png|ico|gif|jpg|jpeg|svg|webp|css|js|xml|json|rss|pdf|zip|woff2?|ttf|eot)"
|
||||
r"([?#]|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# 明显非文章路径
|
||||
_SKIP_PATH = re.compile(
|
||||
r"/(manifest|robots|sitemap|_next/static|__nextjs_|favicon)"
|
||||
r"[/.]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _is_valid_article_url(url: str, source: SourceConfig) -> bool:
|
||||
"""校验 RSS 条目 URL 是否为有效文章链接。
|
||||
|
||||
过滤规则:
|
||||
1. 静态资源（JS/CSS/图片/字体/数据文件）
|
||||
2. 技术路径（manifest/_next/static/robots）
|
||||
3. 源 article_url_pattern 不匹配
|
||||
4. 域名不一致（RSS 跨站污染，如 Yahoo Finance RSS 混入 sports/tech）
|
||||
"""
|
||||
if _SKIP_EXT.search(url):
|
||||
logger.debug("RSS 跳过静态资源: %s", url[:80])
|
||||
return False
|
||||
if _SKIP_PATH.search(url):
|
||||
logger.debug("RSS 跳过技术路径: %s", url[:80])
|
||||
return False
|
||||
|
||||
# 源级 article_url_pattern 校验
|
||||
pattern = getattr(source, "article_url_pattern", "")
|
||||
if pattern:
|
||||
try:
|
||||
if not re.search(pattern, url, re.IGNORECASE):
|
||||
logger.debug("RSS 跳过不匹配 article_url_pattern: %s", url[:80])
|
||||
return False
|
||||
except re.error:
|
||||
pass # 正则异常不阻塞
|
||||
|
||||
# 域名一致性校验（RSS 跨站污染防护）
|
||||
# 例外: Google News RSS 的链接是 news.google.com 跳转 URL，跳过域名校验
|
||||
rss_url = getattr(source, "rss_url", "")
|
||||
if rss_url and "news.google.com" not in rss_url:
|
||||
from urllib.parse import urlparse
|
||||
item_domain = (urlparse(url).hostname or "").removeprefix("www.")
|
||||
hp_domain = (urlparse(source.homepage).hostname or "").removeprefix("www.")
|
||||
|
||||
if hp_domain and item_domain:
|
||||
if item_domain != hp_domain and not item_domain.endswith("." + hp_domain):
|
||||
logger.debug(
|
||||
"RSS 跳过跨站 URL: %s (domain=%s, expected=%s)",
|
||||
url[:80], item_domain, hp_domain,
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _parse_date(date_str: str | None) -> str:
|
||||
"""尝试解析常见日期格式，返回 ISO 8601 字符串。"""
|
||||
if not date_str:
|
||||
return datetime.now().isoformat()
|
||||
|
||||
formats = [
|
||||
"%a, %d %b %Y %H:%M:%S %z", # RFC 2822
|
||||
"%a, %d %b %Y %H:%M:%S %Z",
|
||||
"%Y-%m-%dT%H:%M:%S%z", # ISO 8601
|
||||
"%Y-%m-%dT%H:%M:%SZ",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d",
|
||||
]
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(date_str, fmt).isoformat()
|
||||
except ValueError:
|
||||
continue
|
||||
return date_str
|
||||
|
||||
|
||||
# ── Google News RSS 标题清洗 ────────────────────────────
|
||||
|
||||
_KNOWN_SOURCES = [
|
||||
"Reuters", "WSJ", "Bloomberg", "CNBC", "Financial Times",
|
||||
"MarketWatch", "Barron's", "Yahoo Finance", "Investing.com",
|
||||
"Seeking Alpha", "The Economist", "ForexLive", "ZeroHedge",
|
||||
]
|
||||
|
||||
|
||||
def _clean_google_news_title(title: str) -> str:
|
||||
"""Google News RSS 标题格式: 'Article Title - SourceName' → 去掉来源后缀。"""
|
||||
for src in _KNOWN_SOURCES:
|
||||
suffix = f" - {src}"
|
||||
if title.endswith(suffix):
|
||||
return title[: -len(suffix)].strip()
|
||||
# 通用回退：最后一个 " - " 之后可能是来源
|
||||
last_dash = title.rfind(" - ")
|
||||
if last_dash > 0:
|
||||
suffix = title[last_dash + 3:]
|
||||
if len(suffix) < 30 and not suffix.startswith("http"):
|
||||
return title[:last_dash].strip()
|
||||
return title
|
||||
|
||||
|
||||
# ── RSS/Atom 解析 ─────────────────────────────────────
|
||||
|
||||
|
||||
def _extract_rss_items(xml_text: str) -> list[dict[str, Any]]:
|
||||
"""从 RSS/Atom XML 中提取文章条目。
|
||||
|
||||
自动识别：RSS 2.0、Google News RSS、Atom。
|
||||
"""
|
||||
# 清理可能的 BOM
|
||||
if xml_text.startswith(""):
|
||||
xml_text = xml_text[1:]
|
||||
|
||||
root = ElementTree.fromstring(xml_text)
|
||||
items: list[dict[str, Any]] = []
|
||||
|
||||
# 检测是否为 Google News RSS
|
||||
first_item = root.find(".//item")
|
||||
is_google_news = False
|
||||
if first_item is not None:
|
||||
source_el = first_item.find("source")
|
||||
if source_el is not None and source_el.text:
|
||||
is_google_news = True
|
||||
|
||||
# RSS 2.0 / Google News RSS: channel/item
|
||||
for item in root.iter("item"):
|
||||
title = _strip_html(_text(item, "title"))
|
||||
link = _text(item, "link")
|
||||
description = _strip_html(_text(item, "description"))
|
||||
pub_date = _text(item, "pubDate")
|
||||
|
||||
if is_google_news and title:
|
||||
title = _clean_google_news_title(title)
|
||||
# 描述通常比标题更长，含摘要
|
||||
if description and len(description) > len(title):
|
||||
summary = description
|
||||
else:
|
||||
summary = title
|
||||
else:
|
||||
summary = description or title
|
||||
|
||||
if link and title:
|
||||
items.append({
|
||||
"title": title,
|
||||
"url": link.strip(),
|
||||
"summary": summary,
|
||||
"publish_time": _parse_date(pub_date),
|
||||
})
|
||||
|
||||
if items:
|
||||
tag = "Google News RSS" if is_google_news else "RSS 2.0"
|
||||
logger.debug("%s: 提取 %d 条", tag, len(items))
|
||||
return items
|
||||
|
||||
# Atom: feed/entry
|
||||
for entry in root.iter(_ns("entry")):
|
||||
title = _strip_html(_text(entry, _ns("title")))
|
||||
link = _attr(entry, _ns("link"), "href")
|
||||
summary = _strip_html(_text(entry, _ns("summary")))
|
||||
updated = _text(entry, _ns("updated"))
|
||||
if link and title:
|
||||
items.append({
|
||||
"title": title,
|
||||
"url": link.strip(),
|
||||
"summary": summary or title,
|
||||
"publish_time": _parse_date(updated),
|
||||
})
|
||||
|
||||
logger.debug("Atom: 提取 %d 条", len(items))
|
||||
return items
|
||||
|
||||
|
||||
def _text(element: ElementTree.Element, tag: str) -> str:
|
||||
"""安全获取子元素文本。"""
|
||||
child = element.find(tag)
|
||||
return (child.text or "").strip() if child is not None and child.text else ""
|
||||
|
||||
|
||||
def _attr(element: ElementTree.Element, tag: str, attr: str) -> str:
|
||||
"""安全获取子元素属性。"""
|
||||
child = element.find(tag)
|
||||
return (child.get(attr) or "").strip() if child is not None else ""
|
||||
|
||||
|
||||
# ── 主抓取接口 ───────────────────────────────────────
|
||||
|
||||
|
||||
def crawl_rss_source(source: SourceConfig) -> CrawlResult:
|
||||
"""通过 RSS/Atom Feed 抓取单个新闻源。
|
||||
|
||||
支持 Google News RSS 代理模式：
|
||||
- rss_url 为 news.google.com 时自动识别
|
||||
- 标题自动去来源后缀
|
||||
- 不抓取原文（跳过 Crawl4AI），直接用 RSS 摘要入库
|
||||
|
||||
Args:
|
||||
source: 源配置（需含 rss_url 字段）
|
||||
|
||||
Returns:
|
||||
CrawlResult
|
||||
"""
|
||||
rss_url = getattr(source, "rss_url", None)
|
||||
if not rss_url:
|
||||
return CrawlResult(
|
||||
source_id=source.id,
|
||||
source_name=source.name,
|
||||
error="源未配置 rss_url",
|
||||
)
|
||||
|
||||
today = get_news_day()
|
||||
out_dir = Path(f"data/raw/{source.id}/{today}")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
start_time = datetime.now()
|
||||
articles: list[ArticleItem] = []
|
||||
crawl_time = start_time.isoformat()
|
||||
|
||||
try:
|
||||
resp = httpx.get(
|
||||
rss_url,
|
||||
follow_redirects=True,
|
||||
timeout=30.0,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
||||
"AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "application/rss+xml, application/xml, text/xml, */*",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
xml_text = resp.text
|
||||
except Exception as e:
|
||||
logger.error("[%s] RSS 抓取失败: %s", source.id, e)
|
||||
return CrawlResult(
|
||||
source_id=source.id,
|
||||
source_name=source.name,
|
||||
start_time=crawl_time,
|
||||
end_time=datetime.now().isoformat(),
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
# 解析 RSS
|
||||
try:
|
||||
items = _extract_rss_items(xml_text)
|
||||
except Exception as e:
|
||||
logger.error("[%s] RSS 解析失败: %s", source.id, e)
|
||||
return CrawlResult(
|
||||
source_id=source.id,
|
||||
source_name=source.name,
|
||||
start_time=crawl_time,
|
||||
end_time=datetime.now().isoformat(),
|
||||
error=f"RSS 解析失败: {e}",
|
||||
)
|
||||
|
||||
if not items:
|
||||
logger.warning("[%s] RSS 返回 0 条", source.id)
|
||||
return CrawlResult(
|
||||
source_id=source.id,
|
||||
source_name=source.name,
|
||||
total_found=0,
|
||||
start_time=crawl_time,
|
||||
end_time=datetime.now().isoformat(),
|
||||
)
|
||||
|
||||
# 过滤已存在的 URL（增量）
|
||||
index_path = out_dir / "index.jsonl"
|
||||
existing_hashes: set[str] = _load_existing_hashes(index_path)
|
||||
|
||||
# 构造 ArticleItem
|
||||
success = 0
|
||||
for item in items[:source.max_articles_per_run]:
|
||||
url_hash = _compute_url_hash(item["url"])
|
||||
if url_hash in existing_hashes:
|
||||
logger.debug("[%s] 跳过 RSS 已抓取: %s", source.id, item["url"][:80])
|
||||
continue
|
||||
|
||||
# URL 有效性校验（静态资源 / 跨站污染 / 不匹配 article_url_pattern）
|
||||
if not _is_valid_article_url(item["url"], source):
|
||||
continue
|
||||
|
||||
# 保存文章摘要为 Markdown
|
||||
md_content = f"# {item['title']}\n\n"
|
||||
md_content += f"**来源**: {source.name}\n\n"
|
||||
md_content += f"**发布时间**: {item['publish_time']}\n\n"
|
||||
md_content += f"**原文链接**: {item['url']}\n\n"
|
||||
md_content += f"{item['summary']}\n"
|
||||
|
||||
md_path = out_dir / f"{url_hash}.md"
|
||||
md_path.write_text(md_content, encoding="utf-8")
|
||||
html_path = out_dir / f"{url_hash}.html"
|
||||
html_content = f"<html><head><title>{item['title']}</title></head><body>{item['summary']}</body></html>"
|
||||
html_path.write_text(html_content, encoding="utf-8")
|
||||
|
||||
article = ArticleItem(
|
||||
source_id=source.id,
|
||||
source_name=source.name,
|
||||
url=item["url"],
|
||||
url_hash=url_hash,
|
||||
title=item["title"],
|
||||
crawl_time=crawl_time,
|
||||
publish_time=item["publish_time"],
|
||||
html_path=str(html_path),
|
||||
md_path=str(md_path),
|
||||
word_count=len(item["summary"].split()),
|
||||
status="success",
|
||||
)
|
||||
articles.append(article)
|
||||
existing_hashes.add(url_hash)
|
||||
success += 1
|
||||
|
||||
logger.info("[%s] RSS 抓取: %d 条新文章（总共 %d 条）",
|
||||
source.id, success, len(items))
|
||||
|
||||
return CrawlResult(
|
||||
source_id=source.id,
|
||||
source_name=source.name,
|
||||
total_found=len(items),
|
||||
total_success=success,
|
||||
articles=articles,
|
||||
start_time=crawl_time,
|
||||
end_time=datetime.now().isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def _load_existing_hashes(index_path: Path) -> set[str]:
|
||||
"""读取 index.jsonl 中已有 url_hash。"""
|
||||
hashes: set[str] = set()
|
||||
if not index_path.exists():
|
||||
return hashes
|
||||
import json
|
||||
with open(index_path, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
hashes.add(json.loads(line).get("url_hash", ""))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return hashes
|
||||
Reference in New Issue
Block a user