Initial commit

scripts/a-share-research.service

@@ -0,0 +1,25 @@
+[Unit]
+Description=A 股 Deep Research 定时任务调度器
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=pi
+WorkingDirectory=/home/pi/news
+Environment="PATH=/home/pi/.local/bin:/usr/local/bin:/usr/bin:/bin"
+Environment="LANG=en_US.UTF-8"
+Environment="LC_ALL=en_US.UTF-8"
+Environment="PYTHONIOENCODING=utf-8"
+ExecStart=/home/pi/.local/bin/uv run python -m scripts.run_scheduler
+Restart=on-failure
+RestartSec=30
+StandardOutput=append:/home/pi/news/logs/scheduler.log
+StandardError=append:/home/pi/news/logs/scheduler_error.log
+
+# 优雅退出(等当前 pipeline 跑完)
+KillSignal=SIGINT
+TimeoutStopSec=900
+
+[Install]
+WantedBy=multi-user.target

scripts/run_cninfo.py

@@ -0,0 +1,67 @@
+"""cninfo 公告抓取入口。
+
+用法:
+    uv run python -m scripts.run_cninfo                          # 抓取今天公告
+    uv run python -m scripts.run_cninfo --days 3                # 抓取最近 3 天
+    uv run python -m scripts.run_cninfo --max-pages 50          # 最多 50 页
+    uv run python -m scripts.run_cninfo --no-save               # 试跑不存盘
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from datetime import date, timedelta
+from pathlib import Path
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from crawler.cninfo import crawl_announcements
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(sys.stderr, level=level,
+               format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}")
+    log_path = Path("logs") / "cninfo.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
+
+
+def main() -> int:
+    load_dotenv()
+    parser = argparse.ArgumentParser(description="cninfo 巨潮资讯网公告抓取")
+    parser.add_argument("--days", type=int, default=0,
+                        help="抓取最近 N 天的公告,默认读 CNINFO_DAYS_BACK")
+    parser.add_argument("--start", default=None, help="开始日期 YYYY-MM-DD")
+    parser.add_argument("--end", default=None, help="结束日期 YYYY-MM-DD")
+    parser.add_argument("--max-pages", type=int, default=0, help="最大页数")
+    parser.add_argument("--no-save", action="store_true", help="仅抓取不存盘")
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+
+    start = args.start
+    end = args.end
+    if args.days > 0 and not start:
+        start = (date.today() - timedelta(days=args.days)).strftime("%Y-%m-%d")
+    if not end:
+        end = date.today().strftime("%Y-%m-%d")
+
+    results = crawl_announcements(
+        start_date=start,
+        end_date=end,
+        max_pages=args.max_pages or None,
+        save=not args.no_save,
+    )
+
+    if not results:
+        return 1
+    logger.info("cninfo 抓取完成: {} 条", len(results))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_crawler.py

@@ -0,0 +1,81 @@
+"""M1 抓取入口脚本。
+
+用法:
+    uv run python -m scripts.run_crawler
+    uv run python -m scripts.run_crawler --source cls
+    uv run python -m scripts.run_crawler --config configs/sources.yaml --no-save
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig
+from loguru import logger
+
+from crawler import crawl_all, crawl_source, load_crawler_config
+
+
+def _setup_logger(level: str) -> None:
+    """配置 loguru,输出到控制台与 logs/crawler.log。"""
+    logger.remove()
+    logger.add(sys.stderr, level=level, format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}")
+    log_path = Path("logs") / "crawler.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.add(
+        log_path,
+        level="DEBUG",
+        rotation="10 MB",
+        retention=5,
+        encoding="utf-8",
+        enqueue=True,
+    )
+
+
+async def _run(args: argparse.Namespace) -> int:
+    config = load_crawler_config(args.config)
+
+    if args.source:
+        source = next((s for s in config.sources if s.id == args.source), None)
+        if source is None:
+            logger.error("未找到源 id={}", args.source)
+            return 2
+        if not source.enabled:
+            logger.warning("源 {} 在 yaml 中标记为 disabled,本次仍将抓取", source.id)
+
+        browser_config = BrowserConfig(
+            headless=config.settings.headless,
+            user_agent=config.settings.user_agent,
+            verbose=False,
+        )
+        sem = asyncio.Semaphore(config.settings.concurrency)
+        async with AsyncWebCrawler(config=browser_config) as crawler:
+            results = await crawl_source(crawler, source, config.settings, sem, save=not args.no_save)
+    else:
+        results = await crawl_all(config, save=not args.no_save)
+
+    succ = sum(1 for r in results if r.success)
+    total = len(results)
+    rate = succ / max(total, 1)
+    logger.info("抓取结束 总 {} 成功 {} 成功率 {:.0%}", total, succ, rate)
+    # 验收标准: 成功率 >= 90%
+    return 0 if rate >= 0.9 else 1
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="A 股新闻抓取(M1)")
+    parser.add_argument("--config", default="configs/sources.yaml", help="sources.yaml 路径")
+    parser.add_argument("--source", default=None, help="只抓单个源 id,默认抓全部启用源")
+    parser.add_argument("--no-save", action="store_true", help="不写入本地文件,仅试跑")
+    parser.add_argument("--log-level", default="INFO", help="DEBUG/INFO/WARNING/ERROR")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+    return asyncio.run(_run(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_dedup.py

@@ -0,0 +1,198 @@
+"""M3 批量去重入口脚本。
+
+输入: data/processed/{source}/{YYYYMMDD}/*.json (M2 产物)
+输出:
+    - 指纹库:data/dedup/fingerprints.sqlite3
+    - 唯一文章:data/deduped/{YYYYMMDD}/uniques/{url_hash}.json
+    - 重复记录:data/deduped/{YYYYMMDD}/duplicates.jsonl
+
+用法:
+    uv run python -m scripts.run_dedup                       # 处理今日全部源
+    uv run python -m scripts.run_dedup --date 20260616
+    uv run python -m scripts.run_dedup --source sina --date 20260616
+    uv run python -m scripts.run_dedup --reset               # 清空指纹库重新建立
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections import Counter
+from datetime import date
+from pathlib import Path
+
+from loguru import logger
+from pydantic import ValidationError
+
+from dedup import Deduper
+from extractor import Article
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
+    )
+    log_path = Path("logs") / "dedup.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
+
+
+def _load_article(json_path: Path) -> Article | None:
+    try:
+        data = json.loads(json_path.read_text(encoding="utf-8"))
+        return Article.model_validate(data)
+    except (json.JSONDecodeError, ValidationError) as e:
+        logger.warning("跳过无法解析的 article 文件 {}: {}", json_path, e)
+        return None
+
+
+def _list_source_dirs(processed_root: Path) -> list[str]:
+    if not processed_root.is_dir():
+        return []
+    return sorted(p.name for p in processed_root.iterdir() if p.is_dir())
+
+
+def _process_source_day(
+    source_id: str,
+    day: str,
+    processed_root: Path,
+    out_root: Path,
+    deduper: Deduper,
+) -> tuple[int, int, Counter]:
+    """处理单源单日。返回 (uniques, duplicates, layer_counter)。"""
+    src_dir = processed_root / source_id / day
+    if not src_dir.is_dir():
+        logger.info("源 {} 日期 {} 无 processed 目录,跳过", source_id, day)
+        return 0, 0, Counter()
+
+    files = sorted(src_dir.glob("*.json"))
+    if not files:
+        logger.info("源 {} 日期 {} 无文章,跳过", source_id, day)
+        return 0, 0, Counter()
+
+    uniques_dir = out_root / day / "uniques"
+    uniques_dir.mkdir(parents=True, exist_ok=True)
+    dup_log = out_root / day / "duplicates.jsonl"
+
+    uniq_cnt = 0
+    dup_cnt = 0
+    layer_cnt: Counter = Counter()
+
+    with dup_log.open("a", encoding="utf-8") as dup_f:
+        for fp in files:
+            article = _load_article(fp)
+            if article is None:
+                continue
+            result = deduper.ingest(article)
+            if result.is_duplicate:
+                dup_cnt += 1
+                if result.matched_layer is not None:
+                    layer_cnt[result.matched_layer.value] += 1
+                dup_f.write(
+                    json.dumps(
+                        {
+                            "source_id": article.source_id,
+                            "url": article.url,
+                            "url_hash": article.url_hash,
+                            "title": article.title,
+                            "matched_layer": (
+                                result.matched_layer.value
+                                if result.matched_layer
+                                else None
+                            ),
+                            "matched_url": result.matched_url,
+                            "matched_url_hash": result.matched_url_hash,
+                            "matched_title": result.matched_title,
+                            "hamming_distance": result.hamming_distance,
+                        },
+                        ensure_ascii=False,
+                    )
+                    + "\n"
+                )
+            else:
+                uniq_cnt += 1
+                out_path = uniques_dir / f"{article.url_hash}.json"
+                out_path.write_text(article.model_dump_json(indent=2), encoding="utf-8")
+
+    total = uniq_cnt + dup_cnt
+    rate = dup_cnt / max(total, 1)
+    logger.info(
+        "源 {} 日期 {}: 唯一 {} / 重复 {} (重复率 {:.1%}) layers={}",
+        source_id,
+        day,
+        uniq_cnt,
+        dup_cnt,
+        rate,
+        dict(layer_cnt),
+    )
+    return uniq_cnt, dup_cnt, layer_cnt
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="A 股新闻三层去重 (M3)")
+    parser.add_argument("--processed-root", default="data/processed")
+    parser.add_argument("--out-root", default="data/deduped")
+    parser.add_argument("--db", default="data/dedup/fingerprints.sqlite3")
+    parser.add_argument("--source", default=None, help="只处理单源")
+    parser.add_argument(
+        "--date", default=date.today().strftime("%Y%m%d"), help="日期 YYYYMMDD"
+    )
+    parser.add_argument("--simhash-threshold", type=int, default=3)
+    parser.add_argument("--window-days", type=int, default=30)
+    parser.add_argument("--reset", action="store_true", help="处理前清空指纹库")
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+    processed_root = Path(args.processed_root)
+    out_root = Path(args.out_root)
+
+    sources = [args.source] if args.source else _list_source_dirs(processed_root)
+    if not sources:
+        logger.error("{} 下无源目录", processed_root)
+        return 2
+
+    with Deduper(
+        db_path=args.db,
+        simhash_threshold=args.simhash_threshold,
+        time_window_days=args.window_days,
+    ) as deduper:
+        if args.reset:
+            logger.warning("--reset:清空指纹库 {}", args.db)
+            deduper.store.clear()
+
+        # 清掉同日 duplicates.jsonl 避免重复追加(uniques 用 url_hash 文件名,会自然覆盖)
+        dup_log = out_root / args.date / "duplicates.jsonl"
+        if dup_log.exists():
+            dup_log.unlink()
+
+        total_uniq = 0
+        total_dup = 0
+        total_layers: Counter = Counter()
+        for src in sources:
+            u, d, lc = _process_source_day(
+                src, args.date, processed_root, out_root, deduper
+            )
+            total_uniq += u
+            total_dup += d
+            total_layers.update(lc)
+
+        total = total_uniq + total_dup
+        rate = total_dup / max(total, 1)
+        logger.info(
+            "全部完成: 唯一 {} / 重复 {} (重复率 {:.1%}) layers={}",
+            total_uniq,
+            total_dup,
+            rate,
+            dict(total_layers),
+        )
+        # 验收门槛: ≤ 5%
+        return 0 if rate <= 0.05 or total == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_embedding.py

@@ -0,0 +1,293 @@
+"""M5 批量嵌入入口脚本。
+
+输入策略:
+    A. 默认 events 优先 (--input events):
+       data/events/{day}/*.json (M4 ExtractedEvent),用 LLM 摘要 + 事件标签 + 正文组装文本;
+       同时回查对应 article(M2 输出)以拿正文,如查不到则用 ExtractedEvent.event.summary 作为正文。
+    B. articles 模式 (--input articles):
+       data/processed/{source}/{day}/*.json (M2 Article 直出),仅用 title + content。
+    C. deduped 模式 (--input deduped):
+       data/deduped/{day}/uniques/*.json (M3 唯一文章),仅 title + content。
+
+输出:
+    data/embeddings/{day}/{url_hash}.json     (含 vector 完整内容)
+    data/embeddings/{day}/index.jsonl         (扁平摘要,不含向量,便于检索/调试)
+    data/embeddings/{day}/failed.jsonl        (失败列表)
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+import time
+
+# 修复树莓派等环境的 ASCII locale 问题(UnicodeEncodeError)
+os.environ.setdefault("PYTHONUTF8", "1")
+from datetime import date, datetime
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+from loguru import logger
+from pydantic import ValidationError
+
+from embedding import (
+    EmbeddingError,
+    EmbeddingResult,
+    compose_text,
+    make_async_provider,
+)
+from embedding.base import _from_event_dict
+from extractor import Article
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
+    )
+    log_path = Path("logs") / "embedding.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
+
+
+# --------------------------------------------------------------------------- #
+# 输入收集与文本组装
+# --------------------------------------------------------------------------- #
+
+def _list_source_dirs(p: Path) -> list[str]:
+    if not p.is_dir():
+        return []
+    return sorted(d.name for d in p.iterdir() if d.is_dir())
+
+
+def _load_article_by_hash(processed_root: Path, day: str, url_hash: str) -> Article | None:
+    """根据 url_hash 在 data/processed/*/{day}/ 中查 Article。"""
+    for src_dir in processed_root.iterdir():
+        if not src_dir.is_dir():
+            continue
+        candidate = src_dir / day / f"{url_hash}.json"
+        if candidate.is_file():
+            try:
+                return Article.model_validate(
+                    json.loads(candidate.read_text(encoding="utf-8"))
+                )
+            except (json.JSONDecodeError, ValidationError):
+                return None
+    return None
+
+
+def _build_text_from_event(
+    event_path: Path,
+    processed_root: Path,
+    day: str,
+) -> tuple[str, Article, str | None] | None:
+    """从 ExtractedEvent JSON 构造嵌入文本与文章元数据。"""
+    try:
+        obj: dict[str, Any] = json.loads(event_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as e:
+        logger.warning("跳过损坏 event 文件 {}: {}", event_path, e)
+        return None
+
+    article, head, summary = _from_event_dict(obj)
+    # 真正的正文要去 processed/ 找
+    real_article = _load_article_by_hash(processed_root, day, article.url_hash)
+    if real_article is None:
+        # 退化:用 summary 作为正文(质量降级,但仍可嵌入)
+        real_article = article
+        logger.debug("未找到 Article(url_hash={}),用 summary 作为正文兜底", article.url_hash)
+    else:
+        # 用 processed 的真实数据,但保留 event 元数据(time/source 等)
+        real_article = real_article.model_copy(
+            update={"publish_time": article.publish_time or real_article.publish_time}
+        )
+    text = compose_text(real_article, head=head, summary=summary)
+    return text, real_article, summary
+
+
+def _build_text_from_article(article_path: Path) -> tuple[str, Article, str | None] | None:
+    try:
+        obj = json.loads(article_path.read_text(encoding="utf-8"))
+        article = Article.model_validate(obj)
+    except (json.JSONDecodeError, ValidationError) as e:
+        logger.warning("跳过损坏 article 文件 {}: {}", article_path, e)
+        return None
+    return compose_text(article), article, None
+
+
+def _collect_inputs(args: argparse.Namespace) -> list[tuple[Path, str]]:
+    """返回 [(input_file, kind), ...],kind 为 'event'/'article'。"""
+    day = args.date
+    files: list[tuple[Path, str]] = []
+    if args.input == "events":
+        d = Path(args.events_root) / day
+        files = [(p, "event") for p in sorted(d.glob("*.json"))]
+    elif args.input == "deduped":
+        d = Path(args.deduped_root) / day / "uniques"
+        files = [(p, "article") for p in sorted(d.glob("*.json"))]
+    elif args.input == "articles":
+        proc = Path(args.processed_root)
+        srcs = [args.source] if args.source else _list_source_dirs(proc)
+        for src in srcs:
+            sd = proc / src / day
+            if sd.is_dir():
+                files.extend((p, "article") for p in sorted(sd.glob("*.json")))
+    else:
+        raise ValueError(f"未知 --input 模式: {args.input}")
+    return files
+
+
+# --------------------------------------------------------------------------- #
+# 主流程
+# --------------------------------------------------------------------------- #
+
+async def _run(args: argparse.Namespace) -> int:
+    load_dotenv()
+
+    files = _collect_inputs(args)
+    if args.limit:
+        files = files[: args.limit]
+    if not files:
+        logger.error("未发现任何输入文件: {} ({})", args.input, args.date)
+        return 2
+    logger.info("待嵌入文章数: {} (input={})", len(files), args.input)
+
+    # 准备每篇文本
+    prepared: list[tuple[str, Article, str | None]] = []
+    for fp, kind in files:
+        if kind == "event":
+            built = _build_text_from_event(fp, Path(args.processed_root), args.date)
+        else:
+            built = _build_text_from_article(fp)
+        if built is not None:
+            prepared.append(built)
+    if not prepared:
+        logger.error("所有输入文件均无法解析")
+        return 2
+
+    out_dir = Path(args.out_root) / args.date
+    out_dir.mkdir(parents=True, exist_ok=True)
+    index_path = out_dir / "index.jsonl"
+    failed_path = out_dir / "failed.jsonl"
+    for p in (index_path, failed_path):
+        if p.exists():
+            p.unlink()
+
+    started = time.time()
+    succ_cnt = 0
+    fail_cnt = 0
+
+    async with make_async_provider(
+        args.provider,
+        **({"model": args.model} if args.model else {}),
+    ) as provider:
+        logger.info(
+            "Embedding provider={} model={} dim={}",
+            provider.name, provider.model, provider.dim,
+        )
+
+        # 异步分批
+        batch_size = args.batch_size
+        for i in range(0, len(prepared), batch_size):
+            batch = prepared[i : i + batch_size]
+            texts = [t for t, _, _ in batch]
+            try:
+                vectors = await provider.embed_batch(texts)
+            except EmbeddingError as e:
+                logger.warning("批 {} 嵌入失败: {}", i // batch_size, e)
+                for _, art, _ in batch:
+                    fail_cnt += 1
+                    with failed_path.open("a", encoding="utf-8") as f:
+                        f.write(
+                            json.dumps(
+                                {"url_hash": art.url_hash, "url": art.url,
+                                 "error": str(e)},
+                                ensure_ascii=False,
+                            )
+                            + "\n"
+                        )
+                continue
+
+            for (text, article, _summary), vec in zip(batch, vectors, strict=True):
+                if len(vec) != provider.dim:
+                    logger.warning(
+                        "维度不一致 url_hash={} 实际={} 预期={}",
+                        article.url_hash, len(vec), provider.dim,
+                    )
+                result = EmbeddingResult(
+                    url_hash=article.url_hash,
+                    source_id=article.source_id,
+                    title=article.title,
+                    text=text,
+                    vector=vec,
+                    dim=len(vec),
+                    provider=provider.name,
+                    model=provider.model,
+                    embedded_at=datetime.now(),
+                    char_count=len(text),
+                    publish_time=article.publish_time,
+                )
+
+                # 落盘:完整 + index
+                (out_dir / f"{result.url_hash}.json").write_text(
+                    result.model_dump_json(indent=2), encoding="utf-8"
+                )
+                meta = result.model_dump(exclude={"vector", "text"}, mode="json")
+                meta["text_preview"] = text[:60]
+                with index_path.open("a", encoding="utf-8") as f:
+                    f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+                succ_cnt += 1
+            logger.info(
+                "已处理批 {}: 成功 +{}/{} (累计成功 {})",
+                i // batch_size + 1, len(batch), len(batch), succ_cnt,
+            )
+
+    elapsed = time.time() - started
+    total = succ_cnt + fail_cnt
+    rate = succ_cnt / max(total, 1)
+    logger.info(
+        "Embedding 完成: 成功 {}/{} 成功率 {:.1%} 用时 {:.1f}s",
+        succ_cnt, total, rate, elapsed,
+    )
+    return 0 if rate >= 0.95 or total == 0 else 1
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="A 股新闻 Embedding 向量化 (M5)")
+    parser.add_argument(
+        "--input", default="events",
+        choices=["events", "deduped", "articles"],
+        help="输入来源:events (M4) / deduped (M3) / articles (M2)",
+    )
+    parser.add_argument("--events-root", default="data/events")
+    parser.add_argument("--deduped-root", default="data/deduped")
+    parser.add_argument("--processed-root", default="data/processed")
+    parser.add_argument("--source", default=None,
+                        help="--input articles 时按源过滤")
+    parser.add_argument("--out-root", default="data/embeddings")
+    parser.add_argument(
+        "--date", default=date.today().strftime("%Y%m%d"),
+        help="日期 YYYYMMDD,默认今日",
+    )
+    parser.add_argument("--provider", default=None,
+                        help="dashscope (默认) / local-bge,可被 .env 覆盖")
+    parser.add_argument("--model", default=None,
+                        help="嵌入模型名,覆盖 .env 中的 *_EMBEDDING_MODEL")
+    parser.add_argument("--batch-size", type=int, default=10,
+                        help="每批送 embed 的条数(DashScope 上限 10)")
+    parser.add_argument("--limit", type=int, default=0,
+                        help="最多处理 N 篇,0=不限")
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+    return asyncio.run(_run(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_event_extraction.py

@@ -0,0 +1,220 @@
+"""M4 LLM 投资事件抽取批处理。
+
+输入: data/deduped/{YYYYMMDD}/uniques/*.json (M3 唯一文章产物)
+     或者 data/processed/{source}/{YYYYMMDD}/*.json (M2 直出,跳过去重时使用)
+输出: data/events/{YYYYMMDD}/{url_hash}.json (ExtractedEvent)
+     data/events/{YYYYMMDD}/index.jsonl (扁平摘要)
+     data/events/{YYYYMMDD}/failed.jsonl (失败列表)
+
+用法:
+    uv run python -m scripts.run_event_extraction
+    uv run python -m scripts.run_event_extraction --date 20260616
+    uv run python -m scripts.run_event_extraction --provider qwen --model qwen-plus
+    uv run python -m scripts.run_event_extraction --concurrency 5 --limit 10
+    uv run python -m scripts.run_event_extraction --input-root data/processed --no-deduped
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+import time
+from collections import Counter
+from datetime import date
+from pathlib import Path
+
+from dotenv import load_dotenv
+from loguru import logger
+from pydantic import ValidationError
+
+from extractor import Article
+from llm import (
+    ExtractedEvent,
+    LLMCallError,
+    PromptTemplate,
+    extract_event_async,
+    load_llm_config,
+    make_async_client,
+)
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
+    )
+    log_path = Path("logs") / "llm.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
+
+
+def _collect_inputs(
+    input_root: Path,
+    day: str,
+    use_deduped: bool,
+    source_filter: str | None,
+) -> list[Path]:
+    """根据是否走 dedup 选择输入文件清单。"""
+    if use_deduped:
+        # data/deduped/{day}/uniques/*.json
+        d = input_root / day / "uniques"
+        if not d.is_dir():
+            return []
+        return sorted(d.glob("*.json"))
+    # data/processed/{source}/{day}/*.json
+    files: list[Path] = []
+    if source_filter:
+        srcs = [source_filter]
+    else:
+        srcs = sorted(p.name for p in input_root.iterdir() if p.is_dir())
+    for src in srcs:
+        d = input_root / src / day
+        if d.is_dir():
+            files.extend(sorted(d.glob("*.json")))
+    return files
+
+
+def _load_article(p: Path) -> Article | None:
+    try:
+        return Article.model_validate(json.loads(p.read_text(encoding="utf-8")))
+    except (json.JSONDecodeError, ValidationError) as e:
+        logger.warning("跳过无法解析的 article 文件 {}: {}", p, e)
+        return None
+
+
+async def _run(args: argparse.Namespace) -> int:
+    load_dotenv()  # 读 .env 到 os.environ
+    config = load_llm_config(provider=args.provider, model=args.model)
+    logger.info(
+        "LLM provider={} model={} base_url={}",
+        config.provider, config.model, config.base_url,
+    )
+
+    input_root = Path(args.input_root)
+    use_deduped = not args.no_deduped
+    files = _collect_inputs(input_root, args.date, use_deduped, args.source)
+    if args.limit:
+        files = files[: args.limit]
+    if not files:
+        logger.error(
+            "{} 下未发现 {} 的文章(use_deduped={})",
+            input_root, args.date, use_deduped,
+        )
+        return 2
+    logger.info("待处理文章数: {}", len(files))
+
+    out_dir = Path(args.out_root) / args.date
+    out_dir.mkdir(parents=True, exist_ok=True)
+    index_path = out_dir / "index.jsonl"
+    failed_path = out_dir / "failed.jsonl"
+    # 重跑时清掉旧的 jsonl,避免重复追加
+    for p in (index_path, failed_path):
+        if p.exists():
+            p.unlink()
+
+    template = PromptTemplate(args.prompt)
+    semaphore = asyncio.Semaphore(args.concurrency)
+
+    succ_cnt = 0
+    fail_cnt = 0
+    sentiment_cnt: Counter = Counter()
+    layer_attempts: Counter = Counter()
+    started = time.time()
+
+    async with make_async_client(config) as client:
+
+        async def _process(fp: Path) -> tuple[Path, ExtractedEvent | None, str | None]:
+            article = _load_article(fp)
+            if article is None:
+                return fp, None, "无法解析输入"
+            try:
+                event = await extract_event_async(
+                    client=client,
+                    config=config,
+                    article=article,
+                    template=template,
+                    max_attempts=args.max_attempts,
+                    semaphore=semaphore,
+                )
+                return fp, event, None
+            except LLMCallError as e:
+                return fp, None, str(e)
+
+        tasks = [_process(fp) for fp in files]
+        for coro in asyncio.as_completed(tasks):
+            fp, event, err = await coro
+            if event is None:
+                fail_cnt += 1
+                with failed_path.open("a", encoding="utf-8") as f:
+                    f.write(
+                        json.dumps(
+                            {"file": str(fp), "error": err},
+                            ensure_ascii=False,
+                        )
+                        + "\n"
+                    )
+                continue
+
+            succ_cnt += 1
+            sentiment_cnt[event.event.sentiment.value] += 1
+            layer_attempts[event.attempts] += 1
+
+            event_path = out_dir / f"{event.url_hash}.json"
+            event_path.write_text(event.model_dump_json(indent=2), encoding="utf-8")
+
+            meta = event.model_dump(mode="json")
+            # 扁平 index 不含完整 content,但保留 event 字段
+            with index_path.open("a", encoding="utf-8") as f:
+                f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+
+            logger.info(event.short_summary())
+
+    elapsed = time.time() - started
+    total = succ_cnt + fail_cnt
+    rate = succ_cnt / max(total, 1)
+    logger.info(
+        "完成: 成功 {}/{} 成功率 {:.1%} 用时 {:.1f}s 情绪={} 重试分布={}",
+        succ_cnt, total, rate, elapsed, dict(sentiment_cnt), dict(layer_attempts),
+    )
+    # 验收门槛: ≥ 95% JSON 成功率
+    return 0 if rate >= 0.95 or total == 0 else 1
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="A 股 LLM 投资事件抽取 (M4)")
+    parser.add_argument(
+        "--input-root", default="data/deduped",
+        help="输入根目录(默认 dedup 输出);配合 --no-deduped 时改为 data/processed",
+    )
+    parser.add_argument("--no-deduped", action="store_true",
+                        help="跳过 M3,直接读 M2 处理产物")
+    parser.add_argument("--source", default=None, help="--no-deduped 时按源过滤")
+    parser.add_argument("--out-root", default="data/events")
+    parser.add_argument(
+        "--date", default=date.today().strftime("%Y%m%d"),
+        help="日期 YYYYMMDD,默认今日",
+    )
+    parser.add_argument("--provider", default=None,
+                        help="LLM provider: deepseek / qwen,默认读 .env")
+    parser.add_argument("--model", default=None, help="模型名,默认 provider 默认值")
+    parser.add_argument("--concurrency", type=int, default=3,
+                        help="LLM 异步并发上限")
+    parser.add_argument("--max-attempts", type=int, default=3,
+                        help="单篇文章最大重试次数")
+    parser.add_argument("--limit", type=int, default=0,
+                        help="最多处理 N 篇,0=不限制(用于联调)")
+    parser.add_argument("--prompt", default="prompts/event_extraction.md",
+                        help="Prompt 模板路径")
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+    return asyncio.run(_run(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_extractor.py

@@ -0,0 +1,356 @@
+"""M2 批量正文提取入口脚本。
+
+输入: data/raw/{source}/{YYYYMMDD}/index.jsonl  (M1 产物)
+输出: data/processed/{source}/{YYYYMMDD}/{url_hash}.json (Article)
+      data/processed/{source}/{YYYYMMDD}/index.jsonl (扁平元数据,便于检索)
+
+用法:
+    uv run python -m scripts.run_extractor                          # 处理今日所有源
+    uv run python -m scripts.run_extractor --date 20260616
+    uv run python -m scripts.run_extractor --source sina --date 20260616
+    uv run python -m scripts.run_extractor --raw-root data/raw --out-root data/processed
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import date, datetime
+from pathlib import Path
+
+from loguru import logger
+
+from extractor import Article, ExtractError, extract_article
+from extractor.parser import _url_hash
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
+    )
+    log_path = Path("logs") / "extractor.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
+
+
+def _iter_article_records(raw_dir: Path) -> list[dict]:
+    """读取 M1 产物的 index.jsonl,返回所有可处理的记录。
+
+    新闻源: stage=article + success + html_file
+    cninfo:  有 json_file 字段(CninfoItem 格式)
+    """
+    index_path = raw_dir / "index.jsonl"
+    if not index_path.is_file():
+        return []
+    out: list[dict] = []
+    with index_path.open("r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError as e:
+                logger.warning("跳过非法 jsonl 行 in {}: {}", index_path, e)
+                continue
+            # cninfo: 新格式(CninfoItem → json_file)
+            if rec.get("source_id") == "cninfo" or rec.get("json_file"):
+                if rec.get("json_file"):
+                    out.append(rec)
+                continue
+            # 新闻源: 旧格式(CrawlResult → html_file)
+            if rec.get("stage") == "article" and rec.get("success") and rec.get("html_file"):
+                out.append(rec)
+    return out
+
+
+def _process_one(rec: dict, raw_dir: Path, out_dir: Path,
+                  body_xpath_map: dict[str, str] | None = None) -> Article | None:
+    """处理单条记录,失败返回 None。cninfo 等结构化源跳过 GNE。"""
+    src_id = rec.get("source_id", "")
+
+    # cninfo: 新格式(json_file)或旧格式(html_file+meta JSON)直接解析
+    json_file = rec.get("json_file", "")
+    html_file = rec.get("html_file", "")
+    is_cninfo = (src_id == "cninfo" or bool(json_file))
+
+    if is_cninfo:
+        if json_file:
+            json_path = raw_dir / json_file
+            if json_path.is_file():
+                return _process_cninfo_v2(rec, json_path, out_dir)
+            else:
+                logger.warning("cninfo JSON 文件丢失: {}", json_path)
+                return None
+        # 旧格式兼容: html_file + <meta>JSON
+        if html_file:
+            html_path = raw_dir / html_file
+            if html_path.is_file():
+                return _process_cninfo(rec, html_path, out_dir)
+        return None
+
+    html_path = raw_dir / html_file
+    if not html_path or not html_path.is_file():
+        logger.warning("HTML 文件丢失: {}", html_path)
+        return None
+
+    html = html_path.read_text(encoding="utf-8", errors="ignore")
+    extra_config: dict[str, str] = {}
+    if body_xpath_map and src_id in body_xpath_map:
+        extra_config["body_xpath"] = body_xpath_map[src_id]
+    try:
+        article = extract_article(
+            html=html,
+            source_id=src_id,
+            url=rec["url"],
+            extra_config=extra_config or None,
+        )
+    except ExtractError as e:
+        logger.warning("提取失败 {} {}: {}", rec["source_id"], rec["url"], e.reason)
+        return None
+
+    return _save_article(article, out_dir)
+
+
+def _process_cninfo(rec: dict, html_path: Path, out_dir: Path) -> Article | None:
+    """处理 cninfo 公告记录:从 meta JSON 解析结构化数据。"""
+    import re
+    html = html_path.read_text(encoding="utf-8", errors="ignore")
+    # 提取 <meta>{...}</meta> 中的 JSON
+    m = re.search(r"<meta>(.+?)</meta>", html, re.DOTALL)
+    if not m:
+        logger.warning("cninfo HTML 不含 meta JSON: {}", html_path)
+        return None
+    try:
+        meta = json.loads(m.group(1))
+    except json.JSONDecodeError:
+        logger.warning("cninfo meta JSON 解析失败: {}", html_path)
+        return None
+
+    sec_code = (meta.get("secCode") or "").strip()
+    sec_name = (meta.get("secName") or "").strip()
+    ann_type = (meta.get("announcementType") or "").strip()
+    pdf_url = (meta.get("pdfUrl") or "").strip()
+    title = rec.get("title") or meta.get("title") or ""
+
+    # 构建正文:结构化摘要 + 公告类别翻译
+    content_parts = [f"公司: {sec_name}({sec_code})", f"公告标题: {title}"]
+    if ann_type:
+        content_parts.append(f"公告类别编码: {ann_type}")
+    if pdf_url:
+        content_parts.append(f"PDF: {pdf_url}")
+    content = "\n".join(content_parts)
+
+    # 时间
+    fetched = rec.get("fetched_at")
+    publish_time = None
+    if isinstance(fetched, str):
+        try:
+            from datetime import datetime as dt
+            publish_time = dt.fromisoformat(fetched)
+        except ValueError:
+            pass
+
+    article = Article(
+        source_id="cninfo",
+        url=rec.get("url") or "",
+        url_hash=_url_hash(rec.get("url") or ""),
+        title=title,
+        content=content,
+        author=sec_name,
+        source_name="巨潮资讯网",
+        publish_time=publish_time,
+        publish_time_raw=fetched,
+        word_count=len(content),
+    )
+    return _save_article(article, out_dir)
+
+
+def _process_cninfo_v2(rec: dict, json_path: Path, out_dir: Path) -> Article | None:
+    """处理 cninfo v2 格式: 直接读取 CninfoItem JSON 并转换为 Article。"""
+    try:
+        item_data = json.loads(json_path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning("cninfo JSON 读取失败 {}: {}", json_path, e)
+        return None
+
+    stock_code = item_data.get("stock_code", "")
+    stock_name = item_data.get("stock_name", "")
+    title = item_data.get("title", "")
+    content = item_data.get("content", "")
+    publish_time_str = item_data.get("publish_time", "")
+    item_type = item_data.get("item_type", "announcement")
+    url = item_data.get("url", "")
+    extra = item_data.get("extra", {})
+
+    # 类型中文映射
+    type_map = {
+        "announcement": "公告",
+        "research": "投资者调研",
+        "irm": "互动问答",
+    }
+    type_cn = type_map.get(item_type, item_type)
+
+    # 构建正文
+    content_parts = [
+        f"公司: {stock_name}({stock_code})",
+        f"类型: {type_cn}",
+        f"标题: {title}",
+    ]
+    if extra.get("announcement_type"):
+        content_parts.append(f"公告类别: {extra['announcement_type']}")
+    if url:
+        content_parts.append(f"原文链接: {url}")
+    if content:
+        content_parts.append(f"\n正文:\n{content}")
+    full_content = "\n".join(content_parts)
+
+    # 发布时间解析
+    publish_time = None
+    publish_time_raw = publish_time_str
+    if publish_time_str:
+        try:
+            from datetime import datetime as dt
+            publish_time = dt.strptime(publish_time_str, "%Y-%m-%d")
+        except ValueError:
+            try:
+                publish_time = dt.fromisoformat(publish_time_str)
+            except ValueError:
+                pass
+
+    article = Article(
+        source_id="cninfo",
+        url=url,
+        url_hash=_url_hash(url or title),
+        title=title,
+        content=full_content,
+        author=stock_name,
+        source_name="巨潮资讯网",
+        publish_time=publish_time,
+        publish_time_raw=publish_time_raw,
+        word_count=len(full_content),
+        item_type=item_type,
+    )
+    return _save_article(article, out_dir)
+
+
+def _save_article(article: Article, out_dir: Path) -> Article:
+    """保存 Article JSON 并追加 index。"""
+    out_dir.mkdir(parents=True, exist_ok=True)
+    article_path = out_dir / f"{article.url_hash}.json"
+    article_path.write_text(article.model_dump_json(indent=2), encoding="utf-8")
+    flat = article.model_dump(exclude={"content", "images"}, mode="json")
+    flat["article_file"] = article_path.name
+    flat["content_preview"] = article.content[:80]
+    with (out_dir / "index.jsonl").open("a", encoding="utf-8") as f:
+        f.write(json.dumps(flat, ensure_ascii=False) + "\n")
+    return article
+
+
+def _process_source_day(
+    source_id: str,
+    day: str,
+    raw_root: Path,
+    out_root: Path,
+    body_xpath_map: dict[str, str] | None = None,
+) -> tuple[int, int]:
+    """处理单个源单日。返回 (成功数, 总数)。"""
+    raw_dir = raw_root / source_id / day
+    out_dir = out_root / source_id / day
+
+    records = _iter_article_records(raw_dir)
+    if not records:
+        logger.info("源 {} 日期 {} 无可处理记录", source_id, day)
+        return 0, 0
+
+    # 清理同日旧的 index.jsonl,避免重复追加
+    old_index = out_dir / "index.jsonl"
+    if old_index.exists():
+        old_index.unlink()
+
+    succ = 0
+    for rec in records:
+        article = _process_one(rec, raw_dir, out_dir, body_xpath_map)
+        if article is not None:
+            succ += 1
+    total = len(records)
+    rate = succ / max(total, 1)
+    logger.info(
+        "源 {} 日期 {} 提取完成: {}/{} 成功率 {:.0%}",
+        source_id,
+        day,
+        succ,
+        total,
+        rate,
+    )
+    return succ, total
+
+
+def _list_source_dirs(raw_root: Path) -> list[str]:
+    """列出 raw_root 下所有源 id(子目录名)。"""
+    if not raw_root.is_dir():
+        return []
+    return sorted(p.name for p in raw_root.iterdir() if p.is_dir())
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="A 股新闻正文提取 (M2)")
+    parser.add_argument("--raw-root", default="data/raw", help="M1 抓取产物根目录")
+    parser.add_argument("--out-root", default="data/processed", help="M2 提取结果根目录")
+    parser.add_argument("--source", default=None, help="只处理单个源 id,默认全部")
+    parser.add_argument(
+        "--date",
+        default=date.today().strftime("%Y%m%d"),
+        help="处理日期 YYYYMMDD,默认今日",
+    )
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+    raw_root = Path(args.raw_root)
+    out_root = Path(args.out_root)
+
+    sources = [args.source] if args.source else _list_source_dirs(raw_root)
+    if not sources:
+        logger.error("{} 下未发现任何源目录", raw_root)
+        return 2
+
+    # 加载源配置,构建 source_id → body_xpath 映射
+    body_xpath_map: dict[str, str] = {}
+    try:
+        from crawler.config import load_crawler_config
+        cfg = load_crawler_config()
+        for s in cfg.sources:
+            if s.body_xpath:
+                body_xpath_map[s.id] = s.body_xpath
+        if body_xpath_map:
+            logger.info("已加载 body_xpath 配置: {}", dict(body_xpath_map))
+    except Exception:
+        pass
+
+    started = datetime.now()
+    total_succ = 0
+    total_all = 0
+    for src in sources:
+        succ, total = _process_source_day(src, args.date, raw_root, out_root, body_xpath_map)
+        total_succ += succ
+        total_all += total
+
+    elapsed = (datetime.now() - started).total_seconds()
+    rate = total_succ / max(total_all, 1)
+    logger.info(
+        "全部完成: {}/{} 成功率 {:.0%} 用时 {:.1f}s",
+        total_succ,
+        total_all,
+        rate,
+        elapsed,
+    )
+    return 0 if rate >= 0.9 or total_all == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_mcp_server.py

@@ -0,0 +1,88 @@
+"""M8 MCP 服务入口。
+
+Cherry Studio / Claude Code 通过 stdio 协议调用。
+
+用法:
+    uv run python -m scripts.run_mcp_server              # stdio 模式(默认)
+    uv run python -m scripts.run_mcp_server --sse 8765   # HTTP SSE 模式(调试用)
+
+Cherry Studio 配置:
+    {
+      "mcpServers": {
+        "a-share-research": {
+          "command": "uv",
+          "args": ["run", "python", "-m", "scripts.run_mcp_server"],
+          "cwd": "/home/pi/news"
+        }
+      }
+    }
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from loguru import logger
+
+
+def _setup_logger() -> None:
+    logger.remove()
+    log_path = Path("logs") / "mcp_server.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    # MCP stdio 模式下 stderr 会被协议占用,只写文件日志
+    logger.add(
+        log_path,
+        level="DEBUG",
+        rotation="10 MB",
+        retention=5,
+        encoding="utf-8",
+        enqueue=True,
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="A 股 Deep Research MCP 服务")
+    parser.add_argument("--sse", type=int, default=None,
+                        help="启动 HTTP SSE 模式在指定端口(调试用)")
+    parser.add_argument("--host", default="0.0.0.0", help="SSE 监听地址")
+    args = parser.parse_args()
+
+    _setup_logger()
+    logger.info("MCP 服务启动 mode={}", "sse" if args.sse else "stdio")
+
+    if args.sse:
+        _run_sse(args.host, args.sse)
+    else:
+        _run_stdio()
+
+    return 0
+
+
+def _run_stdio() -> None:
+    from mcp_server.tools import mcp  # noqa: E402
+    mcp.run()
+
+
+def _run_sse(host: str, port: int) -> None:
+    from mcp.server.fastmcp import FastMCP  # noqa: E402
+
+    from mcp_server.tools import (  # noqa: E402
+        search_company_news,
+        search_industry_news,
+        search_news,
+        search_sentiment_trend,
+        search_stock_events,
+    )
+
+    sse_mcp = FastMCP(name="A股DeepResearch", host=host, port=port)
+    sse_mcp.tool()(search_news)
+    sse_mcp.tool()(search_company_news)
+    sse_mcp.tool()(search_industry_news)
+    sse_mcp.tool()(search_stock_events)
+    sse_mcp.tool()(search_sentiment_trend)
+    sse_mcp.run(transport="sse")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_qdrant_ingest.py

@@ -0,0 +1,210 @@
+"""M6 Qdrant 批量入库脚本。
+
+输入: data/embeddings/{date}/*.json (M5 产物,含 vector + payload)
+目标: Qdrant Collection a_share_news
+策略: 幂等 upsert(url_hash 做 point ID,同 ID 覆盖不重复计数)
+
+用法:
+    uv run python -m scripts.run_qdrant_ingest                          # 入库今日
+    uv run python -m scripts.run_qdrant_ingest --date 20260616
+    uv run python -m scripts.run_qdrant_ingest --recreate               # 重建 collection + 全量入库
+    uv run python -m scripts.run_qdrant_ingest --memory                 # 内存模式(测试)
+    uv run python -m scripts.run_qdrant_ingest --limit 10               # 试跑 N 条
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+from vectorstore import SearchResult, VectorStore, make_qdrant_client
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
+    )
+    log_path = Path("logs") / "qdrant.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
+
+
+def _load_embedding_result(path: Path) -> dict[str, Any] | None:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning("跳过损坏文件 {}: {}", path, e)
+        return None
+
+
+def _result_to_point(
+    obj: dict[str, Any],
+    events_dir: Path | None = None,
+) -> dict[str, Any] | None:
+    """把 EmbeddingResult JSON dict 转换为 Qdrant Point 格式。
+
+    Payload 包含 title/url/source_id/publish_time/event/计数 等。
+    事件字段优先从 M4 ExtractedEvent 补充(EmbeddingResult 本身不含 event)。
+    """
+    vector = obj.get("vector")
+    if not vector:
+        return None
+    url_hash = obj["url_hash"]
+    payload = {
+        "url_hash": url_hash,
+        "title": obj.get("title") or "",
+        "url": obj.get("url") or "",
+        "source_id": obj.get("source_id") or "",
+        "publish_time": obj.get("publish_time"),
+        "char_count": obj.get("char_count"),
+        "word_count": obj.get("word_count"),
+        "event": {
+            "stock_codes": [],
+            "company_names": [],
+            "industries": [],
+            "sentiment": "neutral",
+            "importance": 1,
+            "event_type": "其他",
+            "summary": "",
+        },
+    }
+    # 优先从 M4 ExtractedEvent JSON 补事件字段
+    ev = _load_event_from_m4(events_dir, url_hash) if events_dir else None
+    if ev is not None:
+        payload["event"] = ev
+    elif "event" in obj and isinstance(obj["event"], dict):
+        ev_src = obj["event"]
+        payload["event"]["stock_codes"] = list(ev_src.get("stock_codes") or [])
+        payload["event"]["company_names"] = list(ev_src.get("company_names") or [])
+        payload["event"]["industries"] = list(ev_src.get("industries") or [])
+        payload["event"]["sentiment"] = ev_src.get("sentiment") or "neutral"
+        payload["event"]["importance"] = ev_src.get("importance") or 1
+        payload["event"]["event_type"] = ev_src.get("event_type") or "其他"
+        payload["event"]["summary"] = ev_src.get("summary") or ""
+    return {"id": url_hash, "vector": vector, "payload": payload}
+
+
+def _load_event_from_m4(events_dir: Path, url_hash: str) -> dict[str, Any] | None:
+    """从 M4 ExtractedEvent JSON 中提取事件 payload 子集。"""
+    event_file = events_dir / f"{url_hash}.json"
+    if not event_file.is_file():
+        return None
+    try:
+        obj = json.loads(event_file.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return None
+    ev_inner = obj.get("event")
+    if not isinstance(ev_inner, dict):
+        return None
+    return {
+        "stock_codes": list(ev_inner.get("stock_codes") or []),
+        "company_names": list(ev_inner.get("company_names") or []),
+        "industries": list(ev_inner.get("industries") or []),
+        "sentiment": ev_inner.get("sentiment") or "neutral",
+        "importance": ev_inner.get("importance") or 1,
+        "event_type": ev_inner.get("event_type") or "其他",
+        "summary": ev_inner.get("summary") or "",
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="A 股新闻 Qdrant 入库 (M6)")
+    parser.add_argument("--embeddings-root", default="data/embeddings")
+    parser.add_argument(
+        "--date", default=date.today().strftime("%Y%m%d"),
+        help="日期 YYYYMMDD,默认今日",
+    )
+    parser.add_argument("--limit", type=int, default=0,
+                        help="最多处理 N 条,0=不限")
+    parser.add_argument("--recreate", action="store_true",
+                        help="入库前删除并重建 collection")
+    parser.add_argument("--host", default=None, help="Qdrant host(HTTP 模式)")
+    parser.add_argument("--port", type=int, default=None, help="Qdrant HTTP port")
+    parser.add_argument("--path", default="data/qdrant_storage",
+                        help="本地文件模式路径(默认,无需 Docker)")
+    parser.add_argument("--collection", default=None, help="Collection 名")
+    parser.add_argument("--memory", action="store_true",
+                        help="内存模式(仅测试)")
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+
+    # 收集文件
+    emb_dir = Path(args.embeddings_root) / args.date
+    events_dir = Path("data/events") / args.date  # M4 产物(补 event 字段)
+    if not emb_dir.is_dir():
+        logger.error("嵌入目录不存在: {}", emb_dir)
+        return 2
+    files = sorted(emb_dir.glob("*.json"))
+    if args.limit:
+        files = files[: args.limit]
+    if not files:
+        logger.error("{} 下无嵌入文件", emb_dir)
+        return 2
+    logger.info("待入库文章数: {} (date={}), events补: {}",
+                len(files), args.date, "yes" if events_dir.is_dir() else "no")
+
+    # 转换
+    points: list[dict[str, Any]] = []
+    for fp in files:
+        obj = _load_embedding_result(fp)
+        if obj is None:
+            continue
+        pt = _result_to_point(obj, events_dir=events_dir if events_dir.is_dir() else None)
+        if pt is not None:
+            points.append(pt)
+    if not points:
+        logger.error("所有嵌入文件均无法解析")
+        return 2
+
+    client = make_qdrant_client(
+        host=args.host, port=args.port, memory=args.memory,
+        path=args.path if not args.host and not args.memory else None,
+    )
+    store = VectorStore(client, collection_name=args.collection)
+
+    # 初始化 collection
+    try:
+        store.init_collection(recreate=args.recreate)
+    except Exception:  # noqa: BLE001 - Qdrant 未启动时会爆连接错误
+        logger.exception("无法连接 Qdrant,请先 docker compose --profile m6 up -d: {}")
+        return 3
+
+    # 入库
+    before = store.count()
+    store.upsert(points)
+    after = store.count()
+
+    logger.info("入库完成: 前 {} -> 后 {} (净增 {})", before, after, after - before)
+
+    # 简单自检:用第一条向量做检索,验证可查回
+    if points and not args.memory:
+        probe = points[0]
+        try:
+            results: list[SearchResult] = store.query(
+                query_vector=probe["vector"], top_k=1,
+            )
+            if results:
+                r = results[0]
+                logger.info("自检 OK: top-1 title={!r} score={:.4f}", r.title[:30], r.score)
+            else:
+                logger.warning("自检:检索返回空")
+        except Exception as e:  # noqa: BLE001
+            logger.warning("自检失败(不阻塞): {}", e)
+
+    store.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_scheduler.py

@@ -0,0 +1,200 @@
+"""M7 定时任务调度入口。
+
+支持两种模式:
+    --once            立刻执行一次全链路
+    (默认)            启动 APScheduler,按 .env 中 SCHEDULE_TIMES 定时执行
+
+用法:
+    uv run python -m scripts.run_scheduler              # 启动定时服务
+    uv run python -m scripts.run_scheduler --once        # 立即执行一次
+    uv run python -m scripts.run_scheduler --once --date 20260616
+    uv run python -m scripts.run_scheduler --once --steps crawler,extractor
+"""
+
+from __future__ import annotations
+
+import argparse
+import signal
+import sys
+from datetime import date, datetime
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from scheduler import STEP_COMMANDS, run_pipeline
+from scheduler.stock_reporter import generate_all_stock_reports
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
+    )
+    log_path = Path("logs") / "scheduler.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logger.add(log_path, level="DEBUG", rotation="10 MB", retention=5, encoding="utf-8")
+
+
+def _parse_schedule_times(raw: str) -> list[tuple[int, int]]:
+    """解析 SCHEDULE_TIMES 环境变量。
+
+    格式: "07:00,12:00,18:00,22:00"
+    返回: [(7,0), (12,0), (18,0), (22,0)]
+    """
+    out: list[tuple[int, int]] = []
+    for part in raw.split(","):
+        part = part.strip()
+        if not part:
+            continue
+        try:
+            h, m = part.split(":")
+            out.append((int(h), int(m)))
+        except ValueError:
+            logger.warning("SCHEDULE_TIMES 格式错误: {!r},跳过", part)
+    return out
+
+
+def _once(args: argparse.Namespace) -> int:
+    """单次执行模式。"""
+    steps = None
+    if args.steps:
+        steps = [s.strip() for s in args.steps.split(",")]
+    run_pipeline(args.date, steps=steps)
+    return 0
+
+
+def _daemon(args: argparse.Namespace) -> int:
+    """守护进程模式(APScheduler)。"""
+    import os
+
+    from apscheduler.schedulers.background import BackgroundScheduler  # noqa: E402
+    from apscheduler.triggers.cron import CronTrigger  # noqa: E402
+
+    times_raw = os.environ.get("SCHEDULE_TIMES", "07:00,12:00,18:00,22:00")
+    times = _parse_schedule_times(times_raw)
+    if not times:
+        logger.error("SCHEDULE_TIMES 为空或全部非法,无法启动定时任务")
+        return 2
+
+    # 找出最早的时间(当天首次运行),仅该次追加日报步骤
+    sorted_times = sorted(times)
+    first_hour, first_minute = sorted_times[0] if sorted_times else (0, 0)
+
+    scheduler = BackgroundScheduler()
+
+    # 包装函数:每次触发时重新计算日期,避免 date.today() 在注册时冻结。
+    def _scheduled_pipeline(steps: list[str] | None = None) -> None:
+        run_pipeline(date.today().strftime("%Y%m%d"), steps=steps)
+
+    for hour, minute in times:
+        trigger = CronTrigger(hour=hour, minute=minute, timezone="Asia/Shanghai")
+        is_first = (hour == first_hour and minute == first_minute)
+        job_kwargs: dict | None = None
+        if is_first:
+            job_kwargs = {
+                "steps": [k for k in STEP_COMMANDS if k not in ("report", "cninfo_crawl", "cninfo_extract", "cninfo_pdf")] + ["report"]
+            }
+        scheduler.add_job(
+            _scheduled_pipeline,
+            trigger=trigger,
+            kwargs=job_kwargs,
+            id=f"pipeline_{hour:02d}{minute:02d}",
+            name=f"全链路 {'+日报' if is_first else ''} {hour:02d}:{minute:02d}",
+        )
+        logger.info("已注册定时任务: {}每天 {:02d}:{:02d}{}", trigger, hour, minute,
+                     " (含日报)" if is_first else "")
+
+    # cninfo 公告管道(可配置,默认 06:30)
+    cninfo_raw = os.environ.get("CNINFO_SCHEDULE_TIME", "06:30")
+    cninfo_parts = cninfo_raw.split(":")
+    cninfo_h, cninfo_m = int(cninfo_parts[0]), int(cninfo_parts[1]) if len(cninfo_parts) > 1 else 0
+    cninfo_trigger = CronTrigger(hour=cninfo_h, minute=cninfo_m, timezone="Asia/Shanghai")
+    cninfo_steps = ["cninfo_crawl", "cninfo_extract", "cninfo_pdf",
+                    "dedup", "llm", "embedding", "qdrant"]
+    scheduler.add_job(
+        _scheduled_pipeline,
+        trigger=cninfo_trigger,
+        kwargs={"steps": cninfo_steps},
+        id="pipeline_cninfo",
+        name=f"cninfo 公告管道 {cninfo_h:02d}:{cninfo_m:02d}",
+    )
+    logger.info("已注册定时任务: cninfo 公告管道 每天 {:02d}:{:02d}", cninfo_h, cninfo_m)
+
+    # 个股日报(可配置,默认 07:30, 设为空可禁用)
+    stock_raw = os.environ.get("STOCK_REPORT_TIME", "07:30")
+    if stock_raw:
+        stock_parts = stock_raw.split(":")
+        stock_h, stock_m = int(stock_parts[0]), int(stock_parts[1]) if len(stock_parts) > 1 else 0
+        stock_trigger = CronTrigger(hour=stock_h, minute=stock_m, timezone="Asia/Shanghai")
+        scheduler.add_job(
+            generate_all_stock_reports,
+            trigger=stock_trigger,
+            id="stock_report",
+            name=f"个股日报 {stock_h:02d}:{stock_m:02d}",
+        )
+        logger.info("已注册定时任务: 个股日报 每天 {:02d}:{:02d}", stock_h, stock_m)
+    else:
+        logger.info("STOCK_REPORT_TIME 为空, 已禁用个股日报")
+
+    # 优雅退出
+    def _shutdown(signum: int, frame: Any) -> None:
+        logger.info("收到信号 {}, 关闭调度器...", signum)
+        scheduler.shutdown(wait=False)
+        raise SystemExit(0)
+
+    signal.signal(signal.SIGINT, _shutdown)
+    signal.signal(signal.SIGTERM, _shutdown)
+
+    scheduler.start()
+    logger.info("调度器已启动,等待触发... (按 Ctrl+C 退出)")
+
+    # 启动时检查是否有因重启/宕机错过的定时任务,30 分钟内补跑
+    now = datetime.now()
+    for hour, minute in times:
+        scheduled = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
+        missed_minutes = (now - scheduled).total_seconds() / 60
+        if 0 < missed_minutes < 30:
+            logger.warning(
+                "检测到错过的定时任务 {:02d}:{:02d} ({} 分钟前),立即补跑一次",
+                hour, minute, int(missed_minutes),
+            )
+            steps = [k for k in STEP_COMMANDS if k != "report"]
+            if (hour, minute) == sorted_times[0]:
+                steps.append("report")
+            run_pipeline(date.today().strftime("%Y%m%d"), steps=steps)
+
+    import contextlib
+
+    with contextlib.suppress(SystemExit, KeyboardInterrupt):
+        # 保持主线程存活,直到收到退出信号
+        signal.pause()
+
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="A 股新闻定时任务 (M7)")
+    parser.add_argument("--once", action="store_true", help="立即执行一次全链路")
+    parser.add_argument(
+        "--date", default=date.today().strftime("%Y%m%d"),
+        help="日期 YYYYMMDD (仅 --once 模式)",
+    )
+    parser.add_argument("--steps", default=None,
+                        help="仅执行指定步骤,逗号分隔 (如 crawler,extractor)")
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+    load_dotenv()
+
+    if args.once:
+        return _once(args)
+    return _daemon(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/run_xwlb.py

@@ -0,0 +1,107 @@
+"""M1 新闻联播 API 抓取脚本。
+
+从 doorcome API /api/xwlbFine/ 获取 AI 精编的新闻联播条目,
+转换为与 Web 抓取兼容的格式(data/raw/xwlb/{date}/),
+供 M2-M6 管道统一处理。
+
+新闻联播晚间播出,始终抓取前一天数据,不依赖 --date 参数。
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+import urllib.request
+from datetime import date, datetime, timedelta
+from pathlib import Path
+
+from loguru import logger
+
+
+def _setup_logger(level: str) -> None:
+    logger.remove()
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="{time:YYYY-MM-DD HH:mm:ss} | {level} | {name} | {message}",
+    )
+
+
+def _url_hash(s: str) -> str:
+    return hashlib.sha1(s.encode("utf-8")).hexdigest()[:16]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="新闻联播 API 抓取 (M1-xwlb)")
+    parser.add_argument("--output-root", default="data/raw")
+    parser.add_argument("--log-level", default="INFO")
+    args = parser.parse_args()
+
+    _setup_logger(args.log_level)
+
+    # 新闻联播晚间播出,始终抓取前一天
+    day_str = (date.today() - timedelta(days=1)).strftime("%Y%m%d")
+    api_url = f"https://api.doorcome.cn/api/xwlbFine/?start_date={day_str}&end_date={day_str}"
+
+    logger.info("请求新闻联播 API: {}", api_url)
+    try:
+        req = urllib.request.Request(api_url)
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            body = json.loads(resp.read().decode("utf-8"))
+    except Exception as e:
+        logger.error("API 请求失败: {}", e)
+        return 2
+
+    raw_news = body.get("data", {}).get("news", [])
+    if not raw_news:
+        logger.warning("{} 无新闻联播数据", day_str)
+        return 0
+
+    # 准备输出目录
+    out_dir = Path(args.output_root) / "xwlb" / day_str
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    index_path = out_dir / "index.jsonl"
+    saved = 0
+
+    for n in raw_news:
+        sid = n.get("daily_sub_id", 0)
+        title = n.get("news_title", "")
+        content = n.get("news_improve", "")
+        news_day = n.get("news_days", day_str)
+
+        fake_url = f"xwlb://{news_day}/{sid}"
+        h = _url_hash(fake_url)
+        html_file = f"{h}.html"
+
+        html_content = f"""<!DOCTYPE html>
+<html><head><meta charset="utf-8"><title>{title}</title></head>
+<body><article><h1>{title}</h1><div class="article-content">{content}</div></article></body>
+</html>"""
+        (out_dir / f"{h}.html").write_text(html_content, encoding="utf-8")
+
+        meta = {
+            "source_id": "xwlb",
+            "stage": "article",
+            "url": fake_url,
+            "success": True,
+            "status_code": 200,
+            "title": title,
+            "error": None,
+            "fetched_at": datetime.now().isoformat(),
+            "attempts": 1,
+            "url_hash": h,
+            "html_file": html_file,
+        }
+        with index_path.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+        saved += 1
+
+    logger.info("新闻联播 {} 抓取完成: {} 条 -> {}", day_str, saved, out_dir)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())