Initial commit

crawler/storage.py

@@ -0,0 +1,98 @@
+"""抓取结果本地保存。
+
+目录结构:
+    {output_root}/{source_id}/{YYYYMMDD}/
+        ├── {url_hash}.html         # 原始 HTML
+        ├── {url_hash}.md           # Crawl4AI 输出的 Markdown
+        └── index.jsonl             # 元数据(每行一条 CrawlResult)
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import date
+from pathlib import Path
+
+from loguru import logger
+
+from .models import CrawlResult
+
+
+def url_hash(url: str) -> str:
+    """对 URL 取 SHA1,取前 16 位作为文件名。"""
+    return hashlib.sha1(url.encode("utf-8")).hexdigest()[:16]
+
+
+def build_output_dir(output_root: str | Path, source_id: str, day: date | None = None) -> Path:
+    """构造保存目录: {output_root}/{source_id}/{YYYYMMDD}/。"""
+    target_day = day or date.today()
+    return Path(output_root) / source_id / target_day.strftime("%Y%m%d")
+
+
+def save_result(
+    result: CrawlResult,
+    output_root: str | Path,
+    day: date | None = None,
+    *,
+    skip_existing: bool = True,
+) -> Path | None:
+    """保存单条抓取结果到本地。
+
+    返回 HTML 文件路径;失败结果只追加 index.jsonl,不写 HTML/MD,返回 None。
+
+    skip_existing=True 时:如果当日该 URL 已保存(HTML 文件已存在),跳过写入,
+    仅记录日志,返回已存在路径。保证增量抓取不会重复保存。
+    """
+    out_dir = build_output_dir(output_root, result.source_id, day)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    h = url_hash(result.url)
+    html_path: Path | None = None
+
+    # 增量去重:如果当日已抓取过同一 URL,跳过
+    existing_html = out_dir / f"{h}.html"
+    if skip_existing and existing_html.exists():
+        logger.debug("跳过重复 URL (今日已抓取): {} -> {}", result.url, h)
+        return existing_html
+
+    if result.success and result.html:
+        html_path = out_dir / f"{h}.html"
+        html_path.write_text(result.html, encoding="utf-8")
+
+        if result.markdown:
+            md_path = out_dir / f"{h}.md"
+            md_path.write_text(result.markdown, encoding="utf-8")
+
+    # 元数据(不含 html/markdown 大字段,避免 jsonl 巨大)
+    meta = result.model_dump(exclude={"html", "markdown"}, mode="json")
+    meta["url_hash"] = h
+    meta["html_file"] = html_path.name if html_path else None
+
+    index_path = out_dir / "index.jsonl"
+    with index_path.open("a", encoding="utf-8") as f:
+        f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+
+    logger.debug("已保存抓取结果: {}", result.short_summary())
+    return html_path
+
+
+def load_seen_urls(source_id: str, output_root: str | Path = "data/raw") -> set[str]:
+    """加载某个源已抓取的所有 URL hash(跨日期累积)。
+
+    读取 {output_root}/{source_id}/seen_urls.txt,每行一个 url_hash。
+    """
+    seen_path = Path(output_root) / source_id / "seen_urls.txt"
+    if not seen_path.is_file():
+        return set()
+    with seen_path.open("r", encoding="utf-8") as f:
+        return {line.strip() for line in f if line.strip()}
+
+
+def mark_url_seen(source_id: str, url: str, output_root: str | Path = "data/raw") -> None:
+    """追加一个已抓取 URL 的 hash 到 seen_urls.txt。"""
+    seen_path = Path(output_root) / source_id / "seen_urls.txt"
+    seen_path.parent.mkdir(parents=True, exist_ok=True)
+    h = url_hash(url)
+    with seen_path.open("a", encoding="utf-8") as f:
+        f.write(h + "\n")