simon
3b44f64f66
- docs: 删除 CLAUDE.md / continuation.md / english-news-plan.md 及旧版 intlnews_usage.*，
统一迁移到 docs/{README,architecture,quickstart,usage,pipeline,configuration,deployment,development,faq}.md
- README: 精简为仓库入口，指向 docs/
- configs/sources.yaml: 更新注释指向新文档
- .env.example: 修正 DashScope Embedding 端点说明
Pipeline 修复：
- dedup/llm/embedding/vectorstore/reporter: 过滤 M2 no_content / 空正文，避免污染下游与 Qdrant
- dedup/pipeline: 改为先写唯一文件再写指纹，避免崩溃导致文章永久丢失
- crawler/orchestrator: sources_crawled 改为“尝试数”，成功数 = crawled - failed
- crawler/storage: write_index_jsonl 从文章路径推断日期，修复跨天/测试路径问题
- scheduler/pipeline: STEP_TIMEOUTS 实际生效（SIGALRM）
- scheduler/reporter: emb_count 排除 index.json；日报跳过无原文事件
- vectorstore/pipeline: payload 增加 source_ids；--recreate --all 时空日期也重建 collection
- app/cli: extract/dedup/translate/embed/index/pipeline 支持 --date；embed/index 支持 --all；crawl 全源失败返回非零
- scripts: domestic_full/crawl_8g/crawl_2g/pipeline 安全加载 .env；M1 全失败不标记且最终退出码=1
145 lines
6.1 KiB
YAML
145 lines
6.1 KiB
YAML
# 英文财经新闻源配置
|
||
# 每个源的字段说明见 docs/architecture.md 与 docs/configuration.md
|
||
|
||
settings:
|
||
concurrency: 5
|
||
request_delay_sec: 2
|
||
user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||
|
||
sources:
|
||
- id: "reuters"
|
||
name: "Reuters"
|
||
enabled: true
|
||
homepage: "https://www.reuters.com/business/"
|
||
article_url_pattern: "/[^/]+/[^/]+/"
|
||
js_render: false
|
||
max_articles_per_run: 30
|
||
anti_bot_mode: "stealth" # DataDome 反爬（Web 回退，大概率失败）
|
||
rss_url: "https://news.google.com/rss/search?q=site:reuters.com+business&hl=en-US&gl=US&ceid=US:en" # Google News RSS 代理（标题+摘要，绕过 DataDome）
|
||
# ⚠️ Google News RSS 链接为 Google 跳转 URL，无法获取 reuters.com 原文 URL
|
||
# rss_crawler.py 自动识别 Google News RSS 格式，标题去 " - Reuters" 后缀
|
||
|
||
- id: "cnbc"
|
||
name: "CNBC"
|
||
enabled: true
|
||
homepage: "https://www.cnbc.com/world/"
|
||
article_url_pattern: "/\\d{4}/\\d{2}/\\d{2}/"
|
||
js_render: true
|
||
max_articles_per_run: 25
|
||
rss_url: "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114" # US Top News and Analysis
|
||
# 其他可用频道（id 参数切换）：
|
||
# id=15837362 → U.S. News
|
||
# id=15839135 → Earnings
|
||
# id=10000116 → Retail
|
||
|
||
- id: "marketwatch"
|
||
name: "MarketWatch"
|
||
enabled: true
|
||
homepage: "https://www.marketwatch.com/latest-news"
|
||
article_url_pattern: "/story/"
|
||
js_render: false
|
||
max_articles_per_run: 25
|
||
rss_url: "https://feeds.marketwatch.com/marketwatch/topstories" # RSS 优先，绕过 DataDome
|
||
anti_bot_mode: "stealth" # RSS 失败回退（headful 在无头服务器不可用）
|
||
# 其他可用频道：
|
||
# https://feeds.marketwatch.com/marketwatch/marketpulse → MarketPulse
|
||
# https://feeds.marketwatch.com/marketwatch/bulletins → Bulletins
|
||
|
||
- id: "ft"
|
||
name: "Financial Times"
|
||
enabled: true
|
||
homepage: "https://www.ft.com/world"
|
||
article_url_pattern: "/content/"
|
||
js_render: false
|
||
max_articles_per_run: 20
|
||
rss_url: "https://www.ft.com/rss/home" # FT International Homepage RSS
|
||
# 其他可用频道：
|
||
# https://www.ft.com/rss/world
|
||
# https://www.ft.com/rss/companies
|
||
# https://www.ft.com/rss/markets
|
||
# ⚠️ RSS 含摘要+缩略图，但全文仍需 Crawl4AI 抓取（FT 付费墙）
|
||
|
||
- id: "yahoo_finance"
|
||
name: "Yahoo Finance"
|
||
enabled: true
|
||
homepage: "https://finance.yahoo.com/news/"
|
||
article_url_pattern: "/(news|articles)/" # /news/、/economy/.../articles/、/markets/.../articles/
|
||
js_render: true
|
||
max_articles_per_run: 30
|
||
rss_url: "https://news.yahoo.com/rss/finance" # Yahoo Finance RSS
|
||
|
||
- id: "investing"
|
||
name: "Investing.com"
|
||
enabled: true
|
||
homepage: "https://www.investing.com/news/latest-news"
|
||
article_url_pattern: "/news/"
|
||
js_render: true
|
||
max_articles_per_run: 25
|
||
rss_url: "https://news.google.com/rss/search?q=site:investing.com+news&hl=en-US&gl=US&ceid=US:en" # Google News RSS 代理（绕过 Cloudflare，仅摘要）
|
||
anti_bot_mode: "stealth" # Cloudflare 反爬（RSS Google News 代理绕过，Web headful 兜底）
|
||
# ⚠️ Google News RSS 仅获取标题+摘要，全文需 Web 回退 headful Playwright 抓取
|
||
|
||
- id: "seekingalpha"
|
||
name: "Seeking Alpha"
|
||
enabled: true
|
||
homepage: "https://seekingalpha.com/market-news"
|
||
article_url_pattern: "/news/"
|
||
js_render: true
|
||
max_articles_per_run: 20
|
||
anti_bot_mode: "stealth" # 首页 OK，文章页 PerimeterX 拦截 → stealth
|
||
rss_url: "https://seekingalpha.com/feed.xml" # 全站文章 RSS（标题+摘要+链接）
|
||
|
||
- id: "barrons"
|
||
name: "Barron's"
|
||
enabled: true
|
||
homepage: "https://www.barrons.com/latest-news"
|
||
article_url_pattern: "/articles/"
|
||
js_render: false
|
||
max_articles_per_run: 20
|
||
anti_bot_mode: "stealth" # HTTP 403 → stealth 回退
|
||
# ⚠️ 无 Barron's 专属原生 RSS（2026-06-21 实测：feeds.content.dowjones.io/rss/barrons→404，feeds.a.dj.com/rss/RSSBarrons.xml→403）
|
||
# 替代：使用 Dow Jones RSS World News 作为兜底
|
||
rss_url: "https://feeds.a.dj.com/rss/RSSWorldNews.xml" # Dow Jones World News（非 Barron's 专属）
|
||
|
||
- id: "wsj"
|
||
name: "Wall Street Journal"
|
||
enabled: true
|
||
homepage: "https://www.wsj.com/economy"
|
||
article_url_pattern: "/articles/"
|
||
js_render: false
|
||
max_articles_per_run: 20
|
||
anti_bot_mode: "stealth" # DataDome → RSS 优先 + stealth 回退
|
||
rss_url: "https://feeds.a.dj.com/rss/RSSWorldNews.xml" # WSJ World News
|
||
# 其他可用频道：
|
||
# https://feeds.a.dj.com/rss/RSSMarketsMain.xml → Markets
|
||
# https://feeds.a.dj.com/rss/RSSWSJD.xml → WSJ Digital (付费墙内)
|
||
|
||
- id: "economist"
|
||
name: "The Economist"
|
||
enabled: true # ✅ RSS 可用（2026-06-21 验证），RSS 优先；Crawl4AI 回退
|
||
homepage: "https://www.economist.com/finance-and-economics"
|
||
article_url_pattern: "/finance-and-economics/\\d{4}/\\d{2}/\\d{2}/"
|
||
js_render: true
|
||
max_articles_per_run: 15
|
||
anti_bot_mode: "stealth"
|
||
rss_url: "https://www.economist.com/finance-and-economics/rss.xml" # Finance & economics RSS（含标题+摘要+链接）
|
||
|
||
- id: "investinglive"
|
||
name: "InvestingLive"
|
||
enabled: true
|
||
homepage: "https://investinglive.com/"
|
||
article_url_pattern: "/(news|central-banks|commodities|stocks|technical-analysis)/"
|
||
js_render: true
|
||
max_articles_per_run: 25
|
||
anti_bot_mode: "stealth"
|
||
rss_url: "https://investinglive.com/feed" # RSS 含完整正文（~120KB/次，25篇）
|
||
# 2026-07-23 更新: forexlive.com 已 301 重定向到 investinglive.com
|
||
- id: "zerohedge"
|
||
name: "ZeroHedge"
|
||
enabled: true
|
||
homepage: "https://www.zerohedge.com/"
|
||
article_url_pattern: "/(geopolitical|markets|economics|commodities|technology|political|macroeconomics|crypto|energy|health|esg)/"
|
||
js_render: true
|
||
max_articles_per_run: 15
|
||
rss_url: "http://feeds.feedburner.com/zerohedge/feed" # FeedBurner RSS（含正文，~262KB/次）
|