Initial commit
This commit is contained in:
@@ -0,0 +1,962 @@
|
||||
"""A 股 Deep Research 统一 CLI。
|
||||
|
||||
用法:
|
||||
uv run a-share crawl # M1 抓取
|
||||
uv run a-share extract --date 20260616 # M2 提取
|
||||
uv run a-share dedup --date 20260616 # M3 去重
|
||||
uv run a-share events --date 20260616 # M4 LLM 抽取
|
||||
uv run a-share embed --date 20260616 # M5 向量化
|
||||
uv run a-share ingest --date 20260616 # M6 入库
|
||||
uv run a-share pipeline --once # 全链路
|
||||
uv run a-share discover https://xxx.com # 分析站点,推荐配置
|
||||
uv run a-share report # 生成日报
|
||||
uv run a-share search "宁德时代" # 检索知识库
|
||||
uv run a-share status # 状态总览
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 工具
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 操作日志
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
_OP_LOG_PATH = Path("logs") / "operations.log"
|
||||
|
||||
def _log_operation_start(command: str, detail: str = "") -> None:
|
||||
"""记录操作开始。"""
|
||||
_OP_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
line = f"[{ts}] START | {command:20} | {detail}".rstrip()
|
||||
with _OP_LOG_PATH.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def _log_operation_end(command: str, success: bool, elapsed: float, detail: str = "") -> None:
|
||||
"""记录操作结束。"""
|
||||
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
status = "OK" if success else "FAIL"
|
||||
detail_str = f" | {detail}" if detail else ""
|
||||
line = f"[{ts}] END | {command:20} | {status} | {elapsed:.1f}s{detail_str}"
|
||||
with _OP_LOG_PATH.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def _wrap_cmd(name: str, func, *cmd_args: object) -> int:
|
||||
"""包装命令函数,自动记录开始/结束日志。"""
|
||||
from time import perf_counter
|
||||
detail_parts = []
|
||||
for a in cmd_args:
|
||||
if a is not None and a is not False and a != "" and a != 0:
|
||||
detail_parts.append(str(a)[:80])
|
||||
detail = " ".join(detail_parts) if detail_parts else ""
|
||||
_log_operation_start(name, detail)
|
||||
started = perf_counter()
|
||||
try:
|
||||
rc = func()
|
||||
elapsed = perf_counter() - started
|
||||
_log_operation_end(name, rc == 0, elapsed)
|
||||
return rc
|
||||
except Exception as e:
|
||||
elapsed = perf_counter() - started
|
||||
_log_operation_end(name, False, elapsed, f"{type(e).__name__}: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def _setup_logger(level: str = "WARNING") -> None:
|
||||
logger.remove()
|
||||
logger.add(sys.stderr, level=level, format="{level} | {message}")
|
||||
|
||||
|
||||
def _today() -> str:
|
||||
return date.today().strftime("%Y%m%d")
|
||||
|
||||
|
||||
def _resolve_timeout(step_name: str, hardcoded_default: int) -> int:
|
||||
"""解析步骤超时(秒),与 scheduler.pipeline 保持一致的优先级。
|
||||
|
||||
优先级:
|
||||
1. TIMEOUT_{STEP_NAME} 环境变量
|
||||
2. PIPELINE_STEP_TIMEOUT 环境变量 (全局兜底)
|
||||
3. 硬编码默认值 (本函数参数)
|
||||
"""
|
||||
import os
|
||||
specific_key = f"TIMEOUT_{step_name.upper()}"
|
||||
if specific_key in os.environ:
|
||||
return int(os.environ[specific_key])
|
||||
if "PIPELINE_STEP_TIMEOUT" in os.environ:
|
||||
return int(os.environ["PIPELINE_STEP_TIMEOUT"])
|
||||
return hardcoded_default
|
||||
|
||||
|
||||
def _run_module(module: str, extra_args: list[str], timeout: int = 600) -> int:
|
||||
"""调用现有 scripts/run_*.py 模块。"""
|
||||
cmd = ["uv", "run", "python", "-m", module, *extra_args]
|
||||
logger.info("执行: {}", " ".join(cmd))
|
||||
try:
|
||||
result = subprocess.run(cmd, timeout=timeout)
|
||||
return result.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("超时 ({}s): {}", timeout, " ".join(cmd))
|
||||
return 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 子命令
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def cmd_crawl(args: argparse.Namespace) -> int:
|
||||
extra = []
|
||||
if args.source:
|
||||
extra.extend(["--source", args.source])
|
||||
if args.no_save:
|
||||
extra.append("--no-save")
|
||||
return _run_module("scripts.run_crawler", extra, timeout=_resolve_timeout("crawler", 600))
|
||||
|
||||
|
||||
def cmd_extract(args: argparse.Namespace) -> int:
|
||||
extra = ["--date", args.date or _today()]
|
||||
if args.source:
|
||||
extra.extend(["--source", args.source])
|
||||
return _run_module("scripts.run_extractor", extra, timeout=_resolve_timeout("extractor", 300))
|
||||
|
||||
|
||||
def cmd_dedup(args: argparse.Namespace) -> int:
|
||||
extra = ["--date", args.date or _today()]
|
||||
if args.reset:
|
||||
extra.append("--reset")
|
||||
return _run_module("scripts.run_dedup", extra, timeout=_resolve_timeout("dedup", 120))
|
||||
|
||||
|
||||
def cmd_events(args: argparse.Namespace) -> int:
|
||||
extra = ["--date", args.date or _today()]
|
||||
if args.provider:
|
||||
extra.extend(["--provider", args.provider])
|
||||
if args.model:
|
||||
extra.extend(["--model", args.model])
|
||||
if args.limit:
|
||||
extra.extend(["--limit", str(args.limit)])
|
||||
if args.concurrency:
|
||||
extra.extend(["--concurrency", str(args.concurrency)])
|
||||
return _run_module("scripts.run_event_extraction", extra, timeout=_resolve_timeout("llm", 900))
|
||||
|
||||
|
||||
def cmd_embed(args: argparse.Namespace) -> int:
|
||||
extra = ["--date", args.date or _today()]
|
||||
if args.provider:
|
||||
extra.extend(["--provider", args.provider])
|
||||
if args.model:
|
||||
extra.extend(["--model", args.model])
|
||||
return _run_module("scripts.run_embedding", extra, timeout=_resolve_timeout("embedding", 300))
|
||||
|
||||
|
||||
def cmd_ingest(args: argparse.Namespace) -> int:
|
||||
extra = ["--date", args.date or _today()]
|
||||
if args.recreate:
|
||||
extra.append("--recreate")
|
||||
return _run_module("scripts.run_qdrant_ingest", extra, timeout=_resolve_timeout("qdrant", 120))
|
||||
|
||||
|
||||
def cmd_pipeline(args: argparse.Namespace) -> int:
|
||||
pipeline_timeout = _resolve_timeout("pipeline", 3600)
|
||||
if args.cninfo_once:
|
||||
extra = ["--once", "--date", args.date or _today(),
|
||||
"--steps", "cninfo_crawl,cninfo_extract,cninfo_pdf,dedup,llm,embedding,qdrant"]
|
||||
return _run_module("scripts.run_scheduler", extra, timeout=pipeline_timeout)
|
||||
if args.once:
|
||||
extra = ["--once", "--date", args.date or _today()]
|
||||
steps = args.steps
|
||||
if args.report:
|
||||
steps = (steps + ",report") if steps else "report"
|
||||
if steps:
|
||||
extra.extend(["--steps", steps])
|
||||
return _run_module("scripts.run_scheduler", extra, timeout=pipeline_timeout)
|
||||
# 守护进程模式：不应通过 CLI 子进程启动（subprocess.run 无法正确管理后台进程）。
|
||||
# 直接在当前进程启动 APScheduler。
|
||||
print("🔁 启动守护进程模式 (APScheduler) …")
|
||||
print(" 提示: 生产环境请使用 systemd 管理，见 docs/systemd.md")
|
||||
from scripts.run_scheduler import _daemon as _run_daemon
|
||||
# 构造一个简易的 namespace 给 _daemon
|
||||
daemon_args = argparse.Namespace()
|
||||
return _run_daemon(daemon_args)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# search — 直接从终端检索知识库
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def cmd_search(args: argparse.Namespace) -> int:
|
||||
"""嵌入 query → Qdrant 检索 → 格式化输出。"""
|
||||
load_dotenv()
|
||||
_setup_logger("WARNING")
|
||||
|
||||
from embedding import make_sync_provider
|
||||
from vectorstore import SearchFilter, VectorStore, make_qdrant_client
|
||||
|
||||
# 嵌入查询
|
||||
print(f"🔍 检索: {args.query}")
|
||||
emb = make_sync_provider() # 读取 EMBEDDING_PROVIDER 环境变量
|
||||
vec = emb.embed_one(args.query)
|
||||
|
||||
# 构建过滤
|
||||
filt = None
|
||||
has_filter = any([
|
||||
args.source, args.sentiment, args.min_importance,
|
||||
args.stock, args.industry,
|
||||
])
|
||||
if has_filter:
|
||||
kwargs: dict[str, Any] = {}
|
||||
if args.source:
|
||||
kwargs["source_id"] = args.source
|
||||
if args.sentiment:
|
||||
kwargs["sentiment"] = args.sentiment
|
||||
if args.min_importance:
|
||||
kwargs["importance_min"] = args.min_importance
|
||||
if args.stock:
|
||||
# 股票代码可能带或不带后缀,两者都匹配
|
||||
code = args.stock.strip().upper()
|
||||
stock_list = [code] if "." in code else [f"{code}.SZ", f"{code}.SH", f"{code}.BJ"]
|
||||
kwargs["stock_codes"] = stock_list
|
||||
if args.industry:
|
||||
kwargs["industries"] = [args.industry]
|
||||
filt = SearchFilter(**kwargs)
|
||||
|
||||
# 检索
|
||||
client = make_qdrant_client()
|
||||
store = VectorStore(client)
|
||||
hits = store.query(query_vector=vec, top_k=args.top, filter=filt, score_threshold=0.2)
|
||||
store.close()
|
||||
emb.close()
|
||||
|
||||
if not hits:
|
||||
print(f"\n未找到与「{args.query}」相关的结果。")
|
||||
return 0
|
||||
|
||||
print(f"\n共 {len(hits)} 条结果:\n")
|
||||
for i, h in enumerate(hits, 1):
|
||||
ev = h.event or {}
|
||||
sentiment_icon = {"positive": "🟢", "neutral": "⚪", "negative": "🔴"}.get(
|
||||
ev.get("sentiment"), ""
|
||||
)
|
||||
print(f"{i}. {sentiment_icon} {h.title}")
|
||||
print(f" 来源: {h.source_id} | 相似度: {h.score:.4f} | 时间: {h.publish_time}")
|
||||
if ev.get("stock_codes"):
|
||||
print(f" 代码: {','.join(ev['stock_codes'])}")
|
||||
if ev.get("event_type"):
|
||||
print(f" 事件: {ev.get('event_type')} (重要度 {ev.get('importance', '-')})")
|
||||
if ev.get("summary"):
|
||||
print(f" 摘要: {ev['summary']}")
|
||||
print(f" {h.url}")
|
||||
print()
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# status — 数据总览
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def cmd_status(args: argparse.Namespace) -> int: # noqa: ARG001
|
||||
"""输出各层数据统计。"""
|
||||
load_dotenv()
|
||||
today_str = _today()
|
||||
|
||||
def _count_jsonl(path: Path) -> int:
|
||||
if not path.is_file():
|
||||
return 0
|
||||
return sum(1 for _ in open(path, encoding="utf-8"))
|
||||
|
||||
def _count_dir(pattern: str) -> int:
|
||||
return len(list(Path().glob(pattern)))
|
||||
|
||||
def _count_json(pattern: str) -> int:
|
||||
return len(list(Path().glob(pattern)))
|
||||
|
||||
print(f"📊 A 股 Deep Research 状态 — {today_str}")
|
||||
print("─" * 50)
|
||||
|
||||
# M1 raw
|
||||
raw_art = 0
|
||||
for idx in Path("data/raw").glob(f"*/{today_str}/index.jsonl"):
|
||||
raw_art += _count_jsonl(idx)
|
||||
print(f" M1 抓取: {raw_art} 篇原始文章")
|
||||
|
||||
# M2 processed
|
||||
proc = _count_json(f"data/processed/*/{today_str}/*.json")
|
||||
print(f" M2 提取: {proc} 篇正文")
|
||||
|
||||
# M3 deduped
|
||||
deduped = _count_json(f"data/deduped/{today_str}/uniques/*.json")
|
||||
dup_path = Path(f"data/deduped/{today_str}/duplicates.jsonl")
|
||||
dups = _count_jsonl(dup_path) if dup_path.is_file() else 0
|
||||
print(f" M3 去重: {deduped} 篇唯一, {dups} 篇重复")
|
||||
|
||||
# M4 events
|
||||
events = _count_json(f"data/events/{today_str}/*.json")
|
||||
if events > 0:
|
||||
sentiments: Counter = Counter()
|
||||
importances: Counter = Counter()
|
||||
for fp in Path(f"data/events/{today_str}").glob("*.json"):
|
||||
try:
|
||||
obj = json.loads(fp.read_text(encoding="utf-8"))
|
||||
ev = obj.get("event", {})
|
||||
sentiments[ev.get("sentiment", "?")] += 1
|
||||
importances[ev.get("importance", 0)] += 1
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
high = sum(v for k, v in importances.items() if k >= 4)
|
||||
print(f" M4 事件: {events} 篇 "
|
||||
f"(🟢{sentiments.get('positive', 0)} "
|
||||
f"🔴{sentiments.get('negative', 0)} "
|
||||
f"⚪{sentiments.get('neutral', 0)}, "
|
||||
f"重要≥4: {high})")
|
||||
|
||||
# M5 embeddings
|
||||
emb_count = _count_json(f"data/embeddings/{today_str}/*.json")
|
||||
print(f" M5 向量: {emb_count} 条")
|
||||
|
||||
# cninfo 公告
|
||||
cninfo_raw = 0
|
||||
for idx in Path("data/raw/cninfo").glob("*/index.jsonl"):
|
||||
cninfo_raw += _count_jsonl(idx)
|
||||
if cninfo_raw > 0:
|
||||
cninfo_proc = _count_json("data/processed/cninfo/*/*.json")
|
||||
print(f" cninfo: {cninfo_raw} 条公告 (已提取 {cninfo_proc})")
|
||||
else:
|
||||
print(" cninfo: 暂无数据")
|
||||
|
||||
# M6 Qdrant
|
||||
try:
|
||||
from vectorstore import VectorStore, make_qdrant_client
|
||||
c = make_qdrant_client()
|
||||
s = VectorStore(c)
|
||||
total = s.count()
|
||||
s.close()
|
||||
print(f" M6 Qdrant: {total} 条向量")
|
||||
except Exception:
|
||||
print(" M6 Qdrant: 未连接")
|
||||
|
||||
# M7 systemd
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["systemctl", "is-active", "a-share-research"],
|
||||
capture_output=True, text=True, timeout=5,
|
||||
)
|
||||
svc = r.stdout.strip()
|
||||
icon = "✅" if svc == "active" else "❌"
|
||||
print(f" M7 调度: {icon} {svc}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print("─" * 50)
|
||||
|
||||
# 高重要度事件
|
||||
if events > 0:
|
||||
print("\n🔥 今日重要度 ≥4 的事件:")
|
||||
high_events: list[dict] = []
|
||||
for fp in Path(f"data/events/{today_str}").glob("*.json"):
|
||||
try:
|
||||
obj = json.loads(fp.read_text(encoding="utf-8"))
|
||||
ev = obj.get("event", {})
|
||||
if ev.get("importance", 0) >= 4:
|
||||
high_events.append({
|
||||
"title": obj.get("title", ""),
|
||||
"source": obj.get("source_id", ""),
|
||||
"sentiment": ev.get("sentiment", ""),
|
||||
"importance": ev.get("importance", 0),
|
||||
"summary": ev.get("summary", ""),
|
||||
})
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
high_events.sort(key=lambda e: -e["importance"])
|
||||
for e in high_events[:10]:
|
||||
icon = {"positive": "🟢", "negative": "🔴", "neutral": "⚪"}.get(e["sentiment"], "")
|
||||
print(f" {icon} [{e['source']}] {e['title'][:50]} ({e['summary'][:40]})")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# report — 生成每日摘要 HTML 报告并上传
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def cmd_report(args: argparse.Namespace) -> int:
|
||||
"""生成 HTML 日报并上传到 Web 服务器。"""
|
||||
load_dotenv()
|
||||
from scheduler.reporter import generate_report
|
||||
|
||||
day_str = args.date or _today()
|
||||
print(f"📊 生成日报: {day_str}")
|
||||
path = generate_report(day_str, upload=not args.no_upload)
|
||||
if path is None:
|
||||
print("⚠️ 无数据或生成失败")
|
||||
return 1
|
||||
print(f"✅ 日报已保存: {path}")
|
||||
if not args.no_upload:
|
||||
from datetime import date as dt_date
|
||||
today_str = dt_date.today().strftime("%Y%m%d")
|
||||
print(f"✅ 已上传: http://doorcome.cn/echart/research/{today_str}/")
|
||||
return 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# discover — 自动分析站点,生成 sources.yaml 配置建议
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def cmd_discover(args: argparse.Namespace) -> int:
|
||||
"""抓取首页 → 提取链接 → 按 URL 模式聚类 → 输出 yaml 配置。"""
|
||||
import re
|
||||
import urllib.parse
|
||||
from collections import defaultdict
|
||||
|
||||
print(f"🔍 分析站点: {args.url}")
|
||||
print("⏳ 正在抓取首页(含 JS 渲染)...\n")
|
||||
|
||||
# 1. 抓取首页
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
|
||||
|
||||
async def _fetch():
|
||||
bconf = BrowserConfig(headless=True, verbose=False)
|
||||
rconf = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, page_timeout=30000)
|
||||
async with AsyncWebCrawler(config=bconf) as crawler:
|
||||
return await crawler.arun(url=args.url, config=rconf)
|
||||
|
||||
import asyncio
|
||||
result = asyncio.run(_fetch())
|
||||
except ImportError:
|
||||
print("❌ 需要 crawl4ai 依赖,请确保已安装")
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"❌ 抓取失败: {e}")
|
||||
return 1
|
||||
|
||||
if not result or not getattr(result, "success", False):
|
||||
print(f"❌ 页面抓取失败: {getattr(result, 'error_message', 'unknown')}")
|
||||
return 1
|
||||
|
||||
html = getattr(result, "html", "") or ""
|
||||
if not html:
|
||||
print("❌ 页面无 HTML 内容")
|
||||
return 1
|
||||
|
||||
# 2. 提取所有链接
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
parsed_base = urllib.parse.urlparse(args.url)
|
||||
base_domain = f"{parsed_base.scheme}://{parsed_base.netloc}"
|
||||
|
||||
raw_links: list[tuple[str, str]] = []
|
||||
for a in soup.find_all("a", href=True):
|
||||
href = a["href"].strip()
|
||||
if not href or href.startswith(("javascript:", "#", "mailto:", "tel:")):
|
||||
continue
|
||||
absolute = urllib.parse.urljoin(args.url, href).split("#")[0]
|
||||
anchor = (a.get_text() or "").strip()[:40]
|
||||
raw_links.append((absolute, anchor))
|
||||
|
||||
# 3. 过滤出站内链接,排除导航/静态页
|
||||
site_links: list[tuple[str, str]] = []
|
||||
for url, anchor in raw_links:
|
||||
if base_domain not in url:
|
||||
continue
|
||||
# 排除明显的导航/静态页
|
||||
path = urllib.parse.urlparse(url).path.lower()
|
||||
if path in ("/", "") or any(path.endswith(ext) for ext in (".css", ".js", ".png", ".jpg", ".ico", ".svg", ".xml", ".pdf")):
|
||||
continue
|
||||
site_links.append((url, anchor))
|
||||
|
||||
if not site_links:
|
||||
print("❌ 未发现站内链接(可能需要 JS 渲染或页面结构特殊)")
|
||||
print(" 可尝试手动打开浏览器开发者工具查看网络请求")
|
||||
return 1
|
||||
|
||||
unique_links = list(dict.fromkeys(site_links)) # 去重保序
|
||||
|
||||
# 4. 过滤导航/功能页
|
||||
_nav_words = (
|
||||
"about", "download", "feedback", "member", "login", "register", "app",
|
||||
"calendar", "help", "contact", "privacy", "terms", "service", "buy",
|
||||
"markets", "codes", "real", "lives",
|
||||
)
|
||||
article_links: list[tuple[str, str]] = []
|
||||
for url, anchor in unique_links:
|
||||
path_lower = urllib.parse.urlparse(url).path.lower()
|
||||
parts_lower = [p for p in path_lower.split("/") if p]
|
||||
# 跳过明显的导航/功能路径(子串匹配)
|
||||
if any(nav in p for nav in _nav_words for p in parts_lower if len(p) > 2):
|
||||
continue
|
||||
article_links.append((url, anchor))
|
||||
|
||||
if not article_links:
|
||||
# 降级:不过滤
|
||||
article_links = unique_links
|
||||
|
||||
# 5. 按 URL 路径模式聚类
|
||||
clusters: dict[str, list[tuple[str, str]]] = defaultdict(list)
|
||||
for url, anchor in article_links:
|
||||
path = urllib.parse.urlparse(url).path.strip("/")
|
||||
# 把数字/日期/hash 替换为占位符进行聚类
|
||||
pattern = re.sub(r"/\d{4,}", "/{id}", path)
|
||||
pattern = re.sub(r"/[a-f0-9]{32,}", "/{hash}", pattern)
|
||||
pattern = re.sub(r"/\d{4}-\d{2}-\d{2}", "/{date}", pattern)
|
||||
# 恢复目录结构作为聚类键
|
||||
parts = pattern.split("/")
|
||||
# 聚类键:用路径前缀(前两级目录) + 最后一段的模式
|
||||
key = "/".join(parts[:2]) + "/{...}" if len(parts) >= 2 else (parts[0] if parts else "/")
|
||||
clusters[key].append((url, anchor))
|
||||
|
||||
# 过滤只有 1 条的聚类
|
||||
clusters = {k: v for k, v in clusters.items() if len(v) >= 2}
|
||||
if not clusters:
|
||||
print("⚠️ 未发现明显的文章链接模式(每个 URL 路径都不同)")
|
||||
print(" 可手动检查页面结构")
|
||||
return 1
|
||||
|
||||
# 6. 按链接数排序,但优先数字 ID 模式(文章特征)
|
||||
def _score(kv: tuple) -> tuple:
|
||||
_key, _links = kv
|
||||
has_num_id = bool(re.search(r"/\d{4,}", _links[0][0])) # URL 含长数字
|
||||
article_word = any(w in _key.lower() for w in ("article", "news", "detail", "story"))
|
||||
return (has_num_id or article_word, len(_links))
|
||||
|
||||
ranked = sorted(clusters.items(), key=_score, reverse=True)
|
||||
|
||||
# 6. 对每个聚类生成正则
|
||||
site_host = urllib.parse.urlparse(args.url).netloc.replace(".", r"\.")
|
||||
results: list[dict] = []
|
||||
for cluster_key, links in ranked:
|
||||
# 从实际 URL 中提取路径前缀来构建更精确的正则
|
||||
sample_paths = [urllib.parse.urlparse(u).path for u, _ in links[:5]]
|
||||
# 找路径公共前缀
|
||||
common_prefix = _common_path_prefix(sample_paths)
|
||||
# 推测数字部分
|
||||
if re.search(r"/\d+", sample_paths[0]):
|
||||
pattern_re = f"^https?://{site_host}{re.escape(common_prefix)}\\d+"
|
||||
else:
|
||||
pattern_re = f"^https?://{site_host}{re.escape(common_prefix)}.*"
|
||||
# 去掉多余的转义
|
||||
pattern_re = pattern_re.replace(r"\d+", r"\d+")
|
||||
results.append({
|
||||
"key": cluster_key,
|
||||
"count": len(links),
|
||||
"regex": pattern_re,
|
||||
"samples": links[:3],
|
||||
})
|
||||
|
||||
# 7. 输出结果
|
||||
for i, r in enumerate(results, 1):
|
||||
stars = "★★★" if i == 1 else ("★★" if i == 2 else "★")
|
||||
print(f"模式 {chr(64+i)} ({r['count']} 条, 推荐 {stars}):")
|
||||
print(f" 正则: {r['regex']}")
|
||||
print(" 样本:")
|
||||
for url, anchor in r["samples"]:
|
||||
print(f" {url}" + (f" [{anchor}]" if anchor else ""))
|
||||
|
||||
# 8. 输出 yaml 建议
|
||||
best = results[0]
|
||||
source_id = parsed_base.netloc.split(".")[0].replace("-", "_")
|
||||
print()
|
||||
print("─" * 60)
|
||||
print("建议 yaml 配置(复制到 configs/sources.yaml):\n")
|
||||
yaml_snippet = f""" - id: {source_id}
|
||||
name: {source_id}
|
||||
enabled: true
|
||||
homepage: {args.url}
|
||||
article_url_pattern: '{best['regex']}'
|
||||
js_render: {str(not getattr(result, 'markdown', '')).lower() if hasattr(result, 'markdown') else 'true'}
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20"""
|
||||
source_name = args.name or source_id
|
||||
yaml_snippet = yaml_snippet.replace(f"name: {source_id}", f"name: {source_name}")
|
||||
|
||||
# 有额外入口时,追加 extra_homepages 字段
|
||||
if args.extra_urls:
|
||||
extra_lines = "\n".join(f" - {u}" for u in args.extra_urls)
|
||||
yaml_snippet += f"\n extra_homepages:\n{extra_lines}"
|
||||
|
||||
print(yaml_snippet)
|
||||
print("─" * 60)
|
||||
|
||||
if args.add:
|
||||
yaml_path = Path("configs/sources.yaml")
|
||||
if yaml_path.is_file():
|
||||
content = yaml_path.read_text(encoding="utf-8")
|
||||
if f"id: {source_id}" in content:
|
||||
print(f"\n⚠️ sources.yaml 中已存在 id={source_id},跳过添加")
|
||||
else:
|
||||
seq = content.count("\n - id:") + 1
|
||||
header = f"\n # ---- {seq}. {source_name} ----"
|
||||
yaml_path.write_text(content.rstrip() + "\n" + header + "\n" + yaml_snippet + "\n", encoding="utf-8")
|
||||
print(f"\n✅ 已追加到 configs/sources.yaml (第 {seq} 个源)")
|
||||
else:
|
||||
print("\n⚠️ configs/sources.yaml 不存在,无法自动添加")
|
||||
|
||||
print(f"\n验证: uv run a-share crawl --source {source_id}")
|
||||
return 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# add-entry — 为已有源追加 extra_homepages
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def cmd_add_entry(args: argparse.Namespace) -> int:
|
||||
"""给已有源追加 extra_homepages 入口(文本模式,保留 yaml 原有格式)。"""
|
||||
import re
|
||||
|
||||
yaml_path = Path("configs/sources.yaml")
|
||||
if not yaml_path.is_file():
|
||||
print("❌ configs/sources.yaml 不存在")
|
||||
return 1
|
||||
|
||||
content = yaml_path.read_text(encoding="utf-8")
|
||||
|
||||
if f"id: {args.source}" not in content:
|
||||
print(f"❌ 未找到源 id={args.source}")
|
||||
return 1
|
||||
|
||||
# 提取该 source 块中已有的 extra_homepages URL
|
||||
lines = content.splitlines()
|
||||
in_block = False
|
||||
existing: set[str] = set()
|
||||
for line in lines:
|
||||
if f"id: {args.source}" in line and line.strip().startswith("- id:"):
|
||||
in_block = True
|
||||
continue
|
||||
if in_block and (line.strip().startswith("- id:") or line.strip().startswith("settings:")):
|
||||
break
|
||||
m = re.match(r"\s+- (https?://\S+)", line)
|
||||
if m:
|
||||
existing.add(m.group(1))
|
||||
|
||||
added = []
|
||||
seen_this_run: set[str] = set()
|
||||
for url in args.urls:
|
||||
if url in existing or url in seen_this_run:
|
||||
print(f"⏭ 跳过(已存在): {url}")
|
||||
else:
|
||||
seen_this_run.add(url)
|
||||
added.append(url)
|
||||
print(f"✅ 已添加: {url}")
|
||||
|
||||
if not added:
|
||||
print("无新增入口")
|
||||
return 0
|
||||
|
||||
# 在 source 块内插入/追加
|
||||
has_extra_header = any("extra_homepages:" in lines[i] for i in range(len(lines)) if in_block_from(lines, i, args.source))
|
||||
|
||||
if has_extra_header:
|
||||
# 找到块内最后一个 extra URL 行,在其后追加
|
||||
insert_at = -1
|
||||
for i in range(len(lines)):
|
||||
if in_block_from(lines, i, args.source) and re.match(r"\s+- https?://", lines[i]) and "extra_homepages:" in "\n".join(lines[max(0, i - 2):i + 1]):
|
||||
insert_at = i
|
||||
if insert_at > 0:
|
||||
indent = " "
|
||||
for u in added:
|
||||
lines.insert(insert_at + 1, f"{indent}- {u}")
|
||||
insert_at += 1
|
||||
else:
|
||||
# 在 max_articles_per_run 行后插入 extra_homepages
|
||||
for i in range(len(lines)):
|
||||
if in_block_from(lines, i, args.source) and "max_articles_per_run:" in lines[i]:
|
||||
indent = " "
|
||||
new_block = [f"{indent}extra_homepages:"]
|
||||
for u in added:
|
||||
new_block.append(f"{indent} - {u}")
|
||||
for j, nl in enumerate(new_block):
|
||||
lines.insert(i + 1 + j, nl)
|
||||
break
|
||||
|
||||
yaml_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print(f"\n✅ 已写入 configs/sources.yaml ({len(added)} 个新入口)")
|
||||
return 0
|
||||
|
||||
|
||||
def in_block_from(lines: list[str], idx: int, source_id: str) -> bool:
|
||||
"""检查行 idx 是否在 source_id 的配置块内。"""
|
||||
for i in range(idx, -1, -1):
|
||||
if lines[i].strip().startswith(f"- id: {source_id}"):
|
||||
return True
|
||||
if lines[i].strip().startswith("- id:") or lines[i].strip().startswith("settings:"):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# watchlist 管理
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
_WATCHLIST_PATH = Path("configs/watchlist.yaml")
|
||||
|
||||
|
||||
def _load_watchlist() -> list[dict]:
|
||||
import yaml
|
||||
try:
|
||||
with _WATCHLIST_PATH.open(encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
return list(data.get("watchlist") or [])
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _save_watchlist(items: list[dict]) -> None:
|
||||
import yaml
|
||||
_WATCHLIST_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with _WATCHLIST_PATH.open("w", encoding="utf-8") as f:
|
||||
yaml.dump({"watchlist": items}, f, allow_unicode=True, default_flow_style=False, sort_keys=False)
|
||||
|
||||
|
||||
def cmd_watchlist_add(args: argparse.Namespace) -> int:
|
||||
items = _load_watchlist()
|
||||
code = args.code.strip()
|
||||
for it in items:
|
||||
if it.get("code") == code:
|
||||
it["name"] = args.name
|
||||
if args.note:
|
||||
it["note"] = args.note
|
||||
_save_watchlist(items)
|
||||
print(f"✅ 已更新: {code} {args.name}")
|
||||
return 0
|
||||
items.append({"code": code, "name": args.name, "note": args.note or ""})
|
||||
_save_watchlist(items)
|
||||
print(f"✅ 已添加: {code} {args.name}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_watchlist_remove(args: argparse.Namespace) -> int:
|
||||
items = _load_watchlist()
|
||||
code = args.code.strip()
|
||||
new_items = [it for it in items if it.get("code") != code]
|
||||
if len(new_items) == len(items):
|
||||
print(f"⚠️ 未找到: {code}")
|
||||
return 1
|
||||
_save_watchlist(new_items)
|
||||
print(f"✅ 已移除: {code}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_watchlist_list(args: argparse.Namespace) -> int: # noqa: ARG001
|
||||
items = _load_watchlist()
|
||||
if not items:
|
||||
print("📋 关注列表为空")
|
||||
print("添加: uv run a-share watchlist add 000001 平安银行")
|
||||
return 0
|
||||
print(f"📋 关注列表 ({len(items)} 家):")
|
||||
for it in items:
|
||||
note = f" — {it.get('note','')}" if it.get("note") else ""
|
||||
print(f" {it['code']} {it['name']}{note}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_stock_report(args: argparse.Namespace) -> int:
|
||||
"""生成所有关注公司个股日报。"""
|
||||
from scheduler.stock_reporter import generate_all_stock_reports
|
||||
n = generate_all_stock_reports(upload=not args.no_upload)
|
||||
print(f"✅ 个股报告完成: {n} 家")
|
||||
return 0 if n > 0 else 1
|
||||
|
||||
|
||||
def cmd_cninfo(args: argparse.Namespace) -> int:
|
||||
"""cninfo watchlist 抓取(公告+调研+互动易,API 优先)。"""
|
||||
from crawler.cninfo import crawl_watchlist, enrich_articles_with_pdf
|
||||
|
||||
if args.enrich_pdf:
|
||||
n = enrich_articles_with_pdf(day_str=None, limit=args.pdf_limit)
|
||||
print(f"✅ PDF 正文提取完成: {n} 篇")
|
||||
return 0 if n > 0 else 1
|
||||
|
||||
# 默认: watchlist 全类型抓取(公告+调研+互动易)
|
||||
results = crawl_watchlist(save=not args.no_save)
|
||||
|
||||
if not results:
|
||||
print("⚠️ cninfo 抓取: 无数据")
|
||||
return 1
|
||||
|
||||
# 按类型统计
|
||||
ann = sum(1 for it in results if getattr(it, "item_type", "") == "announcement")
|
||||
res = sum(1 for it in results if getattr(it, "item_type", "") == "research")
|
||||
irm = sum(1 for it in results if getattr(it, "item_type", "") == "irm")
|
||||
print(f"✅ cninfo 抓取完成: 公告 {ann} / 调研 {res} / 互动易 {irm} (共 {len(results)} 条)")
|
||||
return 0
|
||||
|
||||
|
||||
def _common_path_prefix(paths: list[str]) -> str:
|
||||
"""找路径列表的公共前缀(逐字符)。"""
|
||||
if not paths:
|
||||
return "/"
|
||||
shortest = min(paths, key=len)
|
||||
for i, ch in enumerate(shortest):
|
||||
if any(p[i:i+1] != ch for p in paths):
|
||||
prefix = shortest[:i]
|
||||
# 截断到最后一个 /
|
||||
last_slash = prefix.rfind("/")
|
||||
return prefix[:last_slash + 1] if last_slash >= 0 else "/"
|
||||
return shortest
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 主入口
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
def main() -> int:
|
||||
load_dotenv()
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="a-share",
|
||||
description="A 股 Deep Research 统一 CLI",
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", help="子命令")
|
||||
|
||||
# ---- crawl ----
|
||||
p = sub.add_parser("crawl", help="M1 新闻抓取")
|
||||
p.add_argument("--source", default=None)
|
||||
p.add_argument("--no-save", action="store_true")
|
||||
p.set_defaults(func=cmd_crawl)
|
||||
|
||||
# ---- extract ----
|
||||
p = sub.add_parser("extract", help="M2 正文提取")
|
||||
p.add_argument("--date", default=None)
|
||||
p.add_argument("--source", default=None)
|
||||
p.set_defaults(func=cmd_extract)
|
||||
|
||||
# ---- dedup ----
|
||||
p = sub.add_parser("dedup", help="M3 三层去重")
|
||||
p.add_argument("--date", default=None)
|
||||
p.add_argument("--reset", action="store_true")
|
||||
p.set_defaults(func=cmd_dedup)
|
||||
|
||||
# ---- events ----
|
||||
p = sub.add_parser("events", help="M4 LLM 事件抽取")
|
||||
p.add_argument("--date", default=None)
|
||||
p.add_argument("--provider", default=None)
|
||||
p.add_argument("--model", default=None)
|
||||
p.add_argument("--limit", type=int, default=0)
|
||||
p.add_argument("--concurrency", type=int, default=0)
|
||||
p.set_defaults(func=cmd_events)
|
||||
|
||||
# ---- embed ----
|
||||
p = sub.add_parser("embed", help="M5 向量化")
|
||||
p.add_argument("--date", default=None)
|
||||
p.add_argument("--provider", default=None)
|
||||
p.add_argument("--model", default=None)
|
||||
p.set_defaults(func=cmd_embed)
|
||||
|
||||
# ---- ingest ----
|
||||
p = sub.add_parser("ingest", help="M6 Qdrant 入库")
|
||||
p.add_argument("--date", default=None)
|
||||
p.add_argument("--recreate", action="store_true")
|
||||
p.set_defaults(func=cmd_ingest)
|
||||
|
||||
# ---- pipeline ----
|
||||
p = sub.add_parser("pipeline", help="M7 全链路 / 定时守护")
|
||||
p.add_argument("--once", action="store_true", help="立即执行一次")
|
||||
p.add_argument("--date", default=None)
|
||||
p.add_argument("--steps", default=None, help="指定步骤(crawler,extractor,...,report)")
|
||||
p.add_argument("--report", action="store_true", help="全链路末尾生成日报")
|
||||
p.add_argument("--cninfo-once", action="store_true", help="cninfo watchlist 全链路(公告+调研+IRM)")
|
||||
p.set_defaults(func=cmd_pipeline)
|
||||
|
||||
# ---- search ----
|
||||
p = sub.add_parser("search", help="检索知识库")
|
||||
p.add_argument("query", help="自然语言查询")
|
||||
p.add_argument("--top", type=int, default=10)
|
||||
p.add_argument("--source", default=None)
|
||||
p.add_argument("--sentiment", default=None, choices=["positive", "negative", "neutral"])
|
||||
p.add_argument("--min-importance", type=int, default=None, dest="min_importance")
|
||||
p.add_argument("--stock", default=None, help="6 位股票代码")
|
||||
p.add_argument("--industry", default=None, help="行业名")
|
||||
p.set_defaults(func=cmd_search)
|
||||
|
||||
# ---- add-entry ----
|
||||
p = sub.add_parser("add-entry", help="为已有源追加 extra_homepages 入口")
|
||||
p.add_argument("source", help="已有源 id(如 wallstreetcn)")
|
||||
p.add_argument("urls", nargs="+", help="额外入口 URL(可多个)")
|
||||
p.set_defaults(func=cmd_add_entry)
|
||||
|
||||
# ---- discover ----
|
||||
p = sub.add_parser("discover", help="分析站点首页,推荐 sources.yaml 配置")
|
||||
p.add_argument("url", help="站点首页 URL(主入口)")
|
||||
p.add_argument("--extra", action="append", default=[], dest="extra_urls",
|
||||
help="额外入口 URL,可重复使用(同站多频道)")
|
||||
p.add_argument("--add", action="store_true", help="自动追加到 configs/sources.yaml")
|
||||
p.add_argument("--name", default=None, help="站点中文名(如 华尔街见闻)")
|
||||
p.set_defaults(func=cmd_discover)
|
||||
|
||||
# ---- report ----
|
||||
p = sub.add_parser("report", help="生成每日 HTML 报告并上传")
|
||||
p.add_argument("--date", default=None)
|
||||
p.add_argument("--no-upload", action="store_true", help="仅生成不上传")
|
||||
p.set_defaults(func=cmd_report)
|
||||
|
||||
# ---- watchlist ----
|
||||
p = sub.add_parser("watchlist", help="管理 cninfo 公告关注列表")
|
||||
sp = p.add_subparsers(dest="wl_cmd")
|
||||
pa = sp.add_parser("add", help="添加关注公司")
|
||||
pa.add_argument("code", help="6 位股票代码")
|
||||
pa.add_argument("name", help="公司简称")
|
||||
pa.add_argument("--note", default="", help="备注")
|
||||
pa.set_defaults(func=cmd_watchlist_add)
|
||||
pr = sp.add_parser("remove", help="移除关注公司")
|
||||
pr.add_argument("code", help="6 位股票代码")
|
||||
pr.set_defaults(func=cmd_watchlist_remove)
|
||||
pl = sp.add_parser("list", help="查看关注列表")
|
||||
pl.set_defaults(func=cmd_watchlist_list)
|
||||
|
||||
# ---- stock-report ----
|
||||
p = sub.add_parser("stock-report", help="生成关注列表个股日报")
|
||||
p.add_argument("--no-upload", action="store_true", help="仅生成不上传")
|
||||
p.set_defaults(func=cmd_stock_report)
|
||||
|
||||
# ---- cninfo ----
|
||||
p = sub.add_parser("cninfo", help="cninfo watchlist 抓取(公告+调研+互动易, API 优先)")
|
||||
p.add_argument("--no-save", action="store_true")
|
||||
p.add_argument("--enrich-pdf", action="store_true", help="下载 PDF 补充正文")
|
||||
p.add_argument("--pdf-limit", type=int, default=50, help="PDF 最多处理 N 篇")
|
||||
p.set_defaults(func=cmd_cninfo)
|
||||
|
||||
# ---- status ----
|
||||
p = sub.add_parser("status", help="数据总览")
|
||||
p.set_defaults(func=cmd_status)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.command is None:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
# 记录操作日志
|
||||
cmd_name = args.command
|
||||
# 提取关键参数作为日志详情
|
||||
detail_parts = []
|
||||
for attr in ["source", "date", "query", "days", "stock", "industry",
|
||||
"sentiment", "once", "cninfo_once", "report"]:
|
||||
val = getattr(args, attr, None)
|
||||
if val is not None and val is not False and val != "" and val != 0:
|
||||
detail_parts.append(f"{attr}={val}")
|
||||
detail = " ".join(detail_parts) if detail_parts else ""
|
||||
|
||||
from time import perf_counter
|
||||
_log_operation_start(cmd_name, detail)
|
||||
started = perf_counter()
|
||||
rc = args.func(args)
|
||||
elapsed = perf_counter() - started
|
||||
_log_operation_end(cmd_name, rc == 0, elapsed)
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user