初始化
This commit is contained in:
+255
@@ -0,0 +1,255 @@
|
||||
"""English Financial News — CLI 入口"""
|
||||
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from dotenv import load_dotenv
|
||||
import typer
|
||||
|
||||
# 加载 .env 环境变量（必须在所有模块导入之前）
|
||||
load_dotenv()
|
||||
|
||||
app = typer.Typer(
|
||||
name="en-news",
|
||||
help="国际财经新闻抓取与深度研究平台",
|
||||
no_args_is_help=True,
|
||||
)
|
||||
|
||||
# ── 日志配置 ──────────────────────────────────────────
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
stream=sys.stderr,
|
||||
)
|
||||
|
||||
# 抑制第三方库噪音日志
|
||||
for _lib in ("httpx", "httpcore", "openai", "urllib3"):
|
||||
logging.getLogger(_lib).setLevel(logging.WARNING)
|
||||
|
||||
logger = logging.getLogger("en-news")
|
||||
|
||||
|
||||
@app.command()
|
||||
def crawl(
|
||||
source: str | None = typer.Option(
|
||||
None, "--source", "-s",
|
||||
help="只抓取指定 source_id（不传则全部）",
|
||||
),
|
||||
profile: str | None = typer.Option(
|
||||
None, "--profile", "-p",
|
||||
help="Profile 名称（2g_headless / 8g_headful）",
|
||||
),
|
||||
):
|
||||
"""M1: 抓取英文财经新闻（部署海外）"""
|
||||
import os
|
||||
from crawler.orchestrator import run_crawl_sync
|
||||
|
||||
if profile:
|
||||
os.environ["EN_NEWS_PROFILE"] = profile
|
||||
logger.info("📋 使用 Profile: %s", profile)
|
||||
|
||||
try:
|
||||
stats = run_crawl_sync(source_filter=source)
|
||||
typer.echo(f"\n✅ 完成: {stats.sources_crawled} 源, {stats.total_articles} 篇文章")
|
||||
if stats.sources_failed:
|
||||
typer.echo(f"⚠️ {stats.sources_failed} 个源有错误")
|
||||
except FileNotFoundError as e:
|
||||
typer.echo(f"❌ {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
except Exception as e:
|
||||
logger.exception("抓取失败")
|
||||
typer.echo(f"❌ 抓取出错: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
@app.command()
|
||||
def extract(
|
||||
source: str | None = typer.Option(
|
||||
None, "--source", "-s",
|
||||
help="只处理指定 source_id（不传则全部）",
|
||||
),
|
||||
):
|
||||
"""M2: 英文正文提取（trafilatura）"""
|
||||
from extractor.pipeline import process_all_sources
|
||||
|
||||
try:
|
||||
stats = process_all_sources(source_filter=source)
|
||||
typer.echo(
|
||||
f"\n✅ 提取: {stats['sources_processed']} 源, "
|
||||
f"{stats['total_articles']} 篇, {stats['elapsed_sec']:.1f}s"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("正文提取失败")
|
||||
typer.echo(f"❌ 提取出错: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
@app.command()
|
||||
def dedup():
|
||||
"""M3: 三层去重"""
|
||||
from dedup.pipeline import dedup_all_sources
|
||||
|
||||
try:
|
||||
stats = dedup_all_sources()
|
||||
typer.echo(
|
||||
f"\n✅ 去重: {stats['sources_processed']} 源, "
|
||||
f"唯一 {stats['unique']} / 重复 {stats['duplicate']} / "
|
||||
f"总计 {stats['total_articles']}, {stats['elapsed_sec']:.1f}s"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("去重失败")
|
||||
typer.echo(f"❌ 去重出错: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
@app.command()
|
||||
def translate():
|
||||
"""M4: 全文翻译 + 投资事件抽取（LLM）"""
|
||||
from llm.pipeline import translate_all_deduped
|
||||
|
||||
try:
|
||||
stats = translate_all_deduped()
|
||||
typer.echo(
|
||||
f"\n✅ 翻译+事件抽取: {stats['success']}/{stats['total']} 篇, "
|
||||
f"{stats['elapsed_sec']:.1f}s ({stats['provider']}/{stats['model']})"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("翻译失败")
|
||||
typer.echo(f"❌ 翻译出错: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
@app.command()
|
||||
def embed():
|
||||
"""M5: 向量生成"""
|
||||
from embedding.pipeline import embed_all_events
|
||||
|
||||
try:
|
||||
stats = embed_all_events()
|
||||
typer.echo(
|
||||
f"\n✅ 向量生成: {stats['success']}/{stats['total']} 篇, "
|
||||
f"{stats['elapsed_sec']:.1f}s ({stats['provider']}/{stats['model']})"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("向量生成失败")
|
||||
typer.echo(f"❌ 向量生成出错: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
@app.command()
|
||||
def index(
|
||||
recreate: bool = typer.Option(
|
||||
False, "--recreate",
|
||||
help="重建 collection（会删除已有数据）",
|
||||
),
|
||||
):
|
||||
"""M6: Qdrant 入库"""
|
||||
from vectorstore.pipeline import get_collection_info, ingest_all_embeddings
|
||||
|
||||
try:
|
||||
stats = ingest_all_embeddings(recreate=recreate)
|
||||
typer.echo(
|
||||
f"\n✅ 入库: {stats['ingested']}/{stats['total']} 条, "
|
||||
f"{stats['elapsed_sec']:.1f}s"
|
||||
)
|
||||
info = get_collection_info()
|
||||
typer.echo(f"📊 Collection: {info['name']} — {info['vectors_count']} 条向量")
|
||||
except Exception as e:
|
||||
logger.exception("入库失败")
|
||||
typer.echo(f"❌ 入库出错: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
@app.command()
|
||||
def search(
|
||||
query: str = typer.Argument(..., help="中文检索查询"),
|
||||
top_k: int = typer.Option(10, help="返回结果数"),
|
||||
):
|
||||
"""M6: Qdrant 语义检索"""
|
||||
from vectorstore.pipeline import search_news
|
||||
|
||||
try:
|
||||
results = search_news(query, top_k=top_k)
|
||||
if not results:
|
||||
typer.echo("🔍 未找到相关结果")
|
||||
return
|
||||
|
||||
typer.echo(f"\n🔍 搜索: {query}\n")
|
||||
for i, r in enumerate(results, 1):
|
||||
typer.echo(f"{i}. [{r.source_id}] score={r.score:.4f}")
|
||||
typer.echo(f" 📰 {r.title_zh or r.title}")
|
||||
if r.events:
|
||||
events_summary = ", ".join(
|
||||
f"{ev.get('event_type','')}({ev.get('sentiment','')})"
|
||||
for ev in r.events[:3]
|
||||
)
|
||||
typer.echo(f" 📌 事件: {events_summary}")
|
||||
typer.echo(f" 🔗 {r.url}")
|
||||
typer.echo()
|
||||
except Exception as e:
|
||||
logger.exception("搜索失败")
|
||||
typer.echo(f"❌ 搜索出错: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
@app.command()
|
||||
def report():
|
||||
"""M7: 日报生成"""
|
||||
from scheduler.reporter import generate_report
|
||||
|
||||
try:
|
||||
path = generate_report()
|
||||
if path:
|
||||
typer.echo(f"\n✅ 日报已生成: {path}")
|
||||
else:
|
||||
typer.echo("⚠️ 无数据，跳过日报生成")
|
||||
except Exception as e:
|
||||
logger.exception("日报生成失败")
|
||||
typer.echo(f"❌ 日报生成出错: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
@app.command()
|
||||
def pipeline(
|
||||
skip_report: bool = typer.Option(
|
||||
False, "--skip-report",
|
||||
help="跳过日报生成",
|
||||
),
|
||||
):
|
||||
"""M7: 一键运行完整管道 M2→M6（+ 可选日报）"""
|
||||
from crawler.utils import get_news_day
|
||||
from scheduler.pipeline import run_pipeline
|
||||
|
||||
date_str = get_news_day()
|
||||
typer.echo(f"🚀 开始全链路管道，日期: {date_str}\n")
|
||||
|
||||
try:
|
||||
result = run_pipeline(date_str, skip_report=skip_report)
|
||||
|
||||
typer.echo(f"\n{'='*50}")
|
||||
typer.echo(f"Pipeline 完成: {result.success_count}/{len(result.steps)} 成功")
|
||||
for s in result.steps:
|
||||
flag = "✅" if s.success else "❌"
|
||||
typer.echo(f" {flag} {s.name}: {s.message} ({s.elapsed_sec:.0f}s)")
|
||||
if result.finished_at and result.started_at:
|
||||
total = (result.finished_at - result.started_at).total_seconds()
|
||||
typer.echo(f"\n⏱ 总耗时: {total:.0f}s")
|
||||
except Exception as e:
|
||||
logger.exception("管道执行失败")
|
||||
typer.echo(f"❌ 管道执行出错: {e}", err=True)
|
||||
raise typer.Exit(code=1)
|
||||
|
||||
|
||||
@app.command()
|
||||
def mcp_server():
|
||||
"""M8: 启动 MCP 服务（供 Claude Code / Cherry Studio 调用）"""
|
||||
from mcp_server.server import mcp
|
||||
|
||||
typer.echo("🚀 启动国际财经 Deep Research MCP 服务...")
|
||||
mcp.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app()
|
||||
Reference in New Issue
Block a user