99 lines
3.4 KiB
Python
99 lines
3.4 KiB
Python
"""抓取结果本地保存。
|
|
|
|
目录结构:
|
|
{output_root}/{source_id}/{YYYYMMDD}/
|
|
├── {url_hash}.html # 原始 HTML
|
|
├── {url_hash}.md # Crawl4AI 输出的 Markdown
|
|
└── index.jsonl # 元数据(每行一条 CrawlResult)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
from loguru import logger
|
|
|
|
from .models import CrawlResult
|
|
|
|
|
|
def url_hash(url: str) -> str:
|
|
"""对 URL 取 SHA1,取前 16 位作为文件名。"""
|
|
return hashlib.sha1(url.encode("utf-8")).hexdigest()[:16]
|
|
|
|
|
|
def build_output_dir(output_root: str | Path, source_id: str, day: date | None = None) -> Path:
|
|
"""构造保存目录: {output_root}/{source_id}/{YYYYMMDD}/。"""
|
|
target_day = day or date.today()
|
|
return Path(output_root) / source_id / target_day.strftime("%Y%m%d")
|
|
|
|
|
|
def save_result(
|
|
result: CrawlResult,
|
|
output_root: str | Path,
|
|
day: date | None = None,
|
|
*,
|
|
skip_existing: bool = True,
|
|
) -> Path | None:
|
|
"""保存单条抓取结果到本地。
|
|
|
|
返回 HTML 文件路径;失败结果只追加 index.jsonl,不写 HTML/MD,返回 None。
|
|
|
|
skip_existing=True 时:如果当日该 URL 已保存(HTML 文件已存在),跳过写入,
|
|
仅记录日志,返回已存在路径。保证增量抓取不会重复保存。
|
|
"""
|
|
out_dir = build_output_dir(output_root, result.source_id, day)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
h = url_hash(result.url)
|
|
html_path: Path | None = None
|
|
|
|
# 增量去重:如果当日已抓取过同一 URL,跳过
|
|
existing_html = out_dir / f"{h}.html"
|
|
if skip_existing and existing_html.exists():
|
|
logger.debug("跳过重复 URL (今日已抓取): {} -> {}", result.url, h)
|
|
return existing_html
|
|
|
|
if result.success and result.html:
|
|
html_path = out_dir / f"{h}.html"
|
|
html_path.write_text(result.html, encoding="utf-8")
|
|
|
|
if result.markdown:
|
|
md_path = out_dir / f"{h}.md"
|
|
md_path.write_text(result.markdown, encoding="utf-8")
|
|
|
|
# 元数据(不含 html/markdown 大字段,避免 jsonl 巨大)
|
|
meta = result.model_dump(exclude={"html", "markdown"}, mode="json")
|
|
meta["url_hash"] = h
|
|
meta["html_file"] = html_path.name if html_path else None
|
|
|
|
index_path = out_dir / "index.jsonl"
|
|
with index_path.open("a", encoding="utf-8") as f:
|
|
f.write(json.dumps(meta, ensure_ascii=False) + "\n")
|
|
|
|
logger.debug("已保存抓取结果: {}", result.short_summary())
|
|
return html_path
|
|
|
|
|
|
def load_seen_urls(source_id: str, output_root: str | Path = "data/raw") -> set[str]:
|
|
"""加载某个源已抓取的所有 URL hash(跨日期累积)。
|
|
|
|
读取 {output_root}/{source_id}/seen_urls.txt,每行一个 url_hash。
|
|
"""
|
|
seen_path = Path(output_root) / source_id / "seen_urls.txt"
|
|
if not seen_path.is_file():
|
|
return set()
|
|
with seen_path.open("r", encoding="utf-8") as f:
|
|
return {line.strip() for line in f if line.strip()}
|
|
|
|
|
|
def mark_url_seen(source_id: str, url: str, output_root: str | Path = "data/raw") -> None:
|
|
"""追加一个已抓取 URL 的 hash 到 seen_urls.txt。"""
|
|
seen_path = Path(output_root) / source_id / "seen_urls.txt"
|
|
seen_path.parent.mkdir(parents=True, exist_ok=True)
|
|
h = url_hash(url)
|
|
with seen_path.open("a", encoding="utf-8") as f:
|
|
f.write(h + "\n")
|