simon/intl_news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初始化

Browse Source
This commit is contained in:
simon
2026-07-18 16:13:52 +08:00
parent c0070f0a5c
commit fe8b417ab6
75 changed files with 12898 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/profiles/2g_headless.yaml
+25
View File
@@ -0,0 +1,25 @@
# ════════════════════════════════════════════════════
# Profile: 2G 轻量 headless stealth
# ════════════════════════════════════════════════════
# 适用场景：
# - 海外服务器（3.5GB 总内存，约 1.2GB 可用）
# - 国内服务器低负载补充抓取
# - 10 个 RSS 优先源 + 2 个 Web 回退源
# 策略：
# - stealth headless（隐藏 webdriver 特征）
# - 串行逐源抓取，内存超限自动 GC
# - 单源 2 小时超时，单页 45 秒超时
# 部署：bash scripts/domestic_crawl_2g.sh [source_id]
# ════════════════════════════════════════════════════
proxy:
enabled: false # 海外服务器无需代理；国内如需，改为 true
crawler:
max_memory_mb: 1800 # 内存上限
source_timeout_sec: 7200 # 单源超时 2h
article_delay_sec: 3.0 # 文章间冷却
page_timeout_sec: 45 # 单页超时
viewport_width: 1024
viewport_height: 768
headful: false # headless stealth
configs/profiles/8g_headful.yaml
+32
View File
@@ -0,0 +1,32 @@
# ════════════════════════════════════════════════════
# Profile: 8G headful + HTTP 代理 → Privoxy → SOCKS5
# ════════════════════════════════════════════════════
# 适用场景：
# - 国内服务器（7.9GB 总内存，约 6.1GB 可用）
# - 所有源均使用 headful Playwright 浏览器直接抓取
# - 通过 Privoxy (HTTP:3128) → ss-local (SOCKS5:1088) 代理访问海外网站
# 策略：
# - headful 模式 + Xvfb 虚拟显示器（更像真人）
# - HTTP 代理 127.0.0.1:3128
# - 更高视口、更长超时
# - 串行逐源（内存充裕但仍需控制并发）
# - 单源 3 小时超时，单页 60 秒超时
# 前置条件：
# sudo apt install xvfb
# Xvfb :99 -screen 0 1280x1024x24 &
# 部署：bash scripts/domestic_crawl_8g.sh [source_id]
# ════════════════════════════════════════════════════
proxy:
enabled: true # 国内服务器通过 HTTP 代理访问海外网站
url: "http://127.0.0.1:3128"
crawler:
max_memory_mb: 7500 # 内存充裕
source_timeout_sec: 10800 # 单源超时 3h（反爬源可能更慢）
article_delay_sec: 4.0 # 文章间冷却更长（更像真人）
page_timeout_sec: 60 # 单页超时 60s
viewport_width: 1280 # 更正常的视口
viewport_height: 900
headful: true # 有头浏览器 + Xvfb
xvfb_display: ":99" # Xvfb 虚拟显示器
configs/sources.yaml
+141
View File
@@ -0,0 +1,141 @@
# 英文财经新闻源配置
# 每个源的字段说明见 english-news-plan.md M1 章节
settings:
concurrency: 5
request_delay_sec: 2
user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
sources:
- id: "reuters"
name: "Reuters"
enabled: true
homepage: "https://www.reuters.com/business/"
article_url_pattern: "/[^/]+/[^/]+/"
js_render: false
max_articles_per_run: 30
anti_bot_mode: "stealth" # DataDome 反爬（Web 回退，大概率失败）
rss_url: "https://news.google.com/rss/search?q=site:reuters.com+business&hl=en-US&gl=US&ceid=US:en" # Google News RSS 代理（标题+摘要，绕过 DataDome）
# ⚠️ Google News RSS 链接为 Google 跳转 URL，无法获取 reuters.com 原文 URL
# rss_crawler.py 自动识别 Google News RSS 格式，标题去 " - Reuters" 后缀
- id: "cnbc"
name: "CNBC"
enabled: true
homepage: "https://www.cnbc.com/world/"
article_url_pattern: "/\\d{4}/\\d{2}/\\d{2}/"
js_render: true
max_articles_per_run: 25
rss_url: "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=100003114" # US Top News and Analysis
# 其他可用频道（id 参数切换）：
# id=15837362 → U.S. News
# id=15839135 → Earnings
# id=10000116 → Retail
- id: "marketwatch"
name: "MarketWatch"
enabled: true
homepage: "https://www.marketwatch.com/latest-news"
article_url_pattern: "/story/"
js_render: false
max_articles_per_run: 25
rss_url: "https://feeds.marketwatch.com/marketwatch/topstories" # RSS 优先，绕过 DataDome
anti_bot_mode: "stealth" # RSS 失败回退（headful 在无头服务器不可用）
# 其他可用频道：
# https://feeds.marketwatch.com/marketwatch/marketpulse → MarketPulse
# https://feeds.marketwatch.com/marketwatch/bulletins → Bulletins
- id: "ft"
name: "Financial Times"
enabled: true
homepage: "https://www.ft.com/world"
article_url_pattern: "/content/"
js_render: false
max_articles_per_run: 20
rss_url: "https://www.ft.com/rss/home" # FT International Homepage RSS
# 其他可用频道：
# https://www.ft.com/rss/world
# https://www.ft.com/rss/companies
# https://www.ft.com/rss/markets
# ⚠️ RSS 含摘要+缩略图，但全文仍需 Crawl4AI 抓取（FT 付费墙）
- id: "yahoo_finance"
name: "Yahoo Finance"
enabled: true
homepage: "https://finance.yahoo.com/news/"
article_url_pattern: "/(news|articles)/" # /news/、/economy/.../articles/、/markets/.../articles/
js_render: true
max_articles_per_run: 30
rss_url: "https://news.yahoo.com/rss/finance" # Yahoo Finance RSS
- id: "investing"
name: "Investing.com"
enabled: true
homepage: "https://www.investing.com/news/latest-news"
article_url_pattern: "/news/"
js_render: true
max_articles_per_run: 25
rss_url: "https://www.investing.com/rss/news_1063.rss" # RSS 绕过反爬
- id: "seekingalpha"
name: "Seeking Alpha"
enabled: true
homepage: "https://seekingalpha.com/market-news"
article_url_pattern: "/news/"
js_render: true
max_articles_per_run: 20
anti_bot_mode: "stealth" # 首页 OK，文章页 PerimeterX 拦截 → stealth
rss_url: "https://seekingalpha.com/feed.xml" # 全站文章 RSS（标题+摘要+链接）
- id: "barrons"
name: "Barron's"
enabled: true
homepage: "https://www.barrons.com/latest-news"
article_url_pattern: "/articles/"
js_render: false
max_articles_per_run: 20
anti_bot_mode: "stealth" # HTTP 403 → stealth 回退
# ⚠️ 无 Barron's 专属原生 RSS（2026-06-21 实测：feeds.content.dowjones.io/rss/barrons→404，feeds.a.dj.com/rss/RSSBarrons.xml→403）
# 替代：使用 Dow Jones RSS World News 作为兜底
rss_url: "https://feeds.a.dj.com/rss/RSSWorldNews.xml" # Dow Jones World News（非 Barron's 专属）
- id: "wsj"
name: "Wall Street Journal"
enabled: true
homepage: "https://www.wsj.com/economy"
article_url_pattern: "/articles/"
js_render: false
max_articles_per_run: 20
anti_bot_mode: "stealth" # DataDome → RSS 优先 + stealth 回退
rss_url: "https://feeds.a.dj.com/rss/RSSWorldNews.xml" # WSJ World News
# 其他可用频道：
# https://feeds.a.dj.com/rss/RSSMarketsMain.xml → Markets
# https://feeds.a.dj.com/rss/RSSWSJD.xml → WSJ Digital (付费墙内)
- id: "economist"
name: "The Economist"
enabled: true # ✅ RSS 可用（2026-06-21 验证），RSS 优先；Crawl4AI 回退
homepage: "https://www.economist.com/finance-and-economics"
article_url_pattern: "/finance-and-economics/\\d{4}/\\d{2}/\\d{2}/"
js_render: true
max_articles_per_run: 15
anti_bot_mode: "stealth"
rss_url: "https://www.economist.com/finance-and-economics/rss.xml" # Finance & economics RSS（含标题+摘要+链接）
- id: "forexlive"
name: "ForexLive"
enabled: true
homepage: "https://www.forexlive.com/"
article_url_pattern: "/(news|technical-analysis|Education)/"
js_render: false
max_articles_per_run: 20
rss_url: "https://www.forexlive.com/feed" # Breaking News RSS（含正文，~91KB/次）
- id: "zerohedge"
name: "ZeroHedge"
enabled: true
homepage: "https://www.zerohedge.com/"
article_url_pattern: "/(geopolitical|markets|economics|commodities|technology|political|macroeconomics|crypto|energy|health|esg)/"
js_render: true
max_articles_per_run: 15
rss_url: "http://feeds.feedburner.com/zerohedge/feed" # FeedBurner RSS（含正文，~262KB/次）
configs/system.yaml
+91
View File
@@ -0,0 +1,91 @@
# ════════════════════════════════════════════════════
# 系统功能配置（非敏感）
# 密钥、API 地址 请在 .env 中配置
# ════════════════════════════════════════════════════
# ── 部署拓扑 ────────────────────────────────────────
# 海外服务器已弃用（M1-M6 全链路在 domestic 执行）
servers:
overseas_host: "ecs-user@8.217.19.253"
overseas_path: "/opt/intlgrab"
domestic_host: "pi@192.168.1.160"
domestic_path: "/home/pi/intlnews"
# ── HTTP 代理（Privoxy → ss-local → Shadowsocks 海外访问）──
proxy:
enabled: false # 通过 Profile 8g_headful 启用
url: "http://127.0.0.1:3128" # HTTP 代理地址
bypass_domains: [] # 不走代理的域名（如国内 CDN）
# ── 抓取 ────────────────────────────────────────────
crawler:
max_memory_mb: 1800 # 内存上限 MB（超限触发 GC）
source_timeout_sec: 7200 # 单源超时秒数（2h）
article_delay_sec: 3.0 # 文章间冷却间隔
page_timeout_sec: 45 # 单页加载超时
viewport_width: 1024
viewport_height: 768
user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
headful: false # true=有头浏览器（需 X Server 或 xvfb），false=headless
xvfb_display: ":99" # headful 模式下的虚拟显示器（xvfb-run 自动设置）
# ── 正文提取 ──────────────────────────────────────────
extractor:
min_content_words: 50
trafilatura_fallback: true
# ── 去重 ────────────────────────────────────────────
dedup:
hamming_distance_threshold: 3
simhash_window_days: 30
min_content_length: 100
# ── LLM 翻译+事件抽取 ────────────────────────────────
llm:
provider: "deepseek"
deepseek_model: "deepseek-chat"
qwen_model: "qwen-plus"
timeout_sec: 60
max_retries: 3
max_tokens: 8192
temperature: 0.1
concurrency: 3
# ── Embedding 向量化 ────────────────────────────────
embedding:
provider: "dashscope"
dashscope_model: "text-embedding-v3"
dimension: 1024
batch_size: 10
timeout_sec: 30
# ── Qdrant ──────────────────────────────────────────
qdrant:
collection: "en_finance_news"
# ── 日报 ────────────────────────────────────────────
report:
max_events: 20
summary_max_chars: 500
importance_threshold: 4
upload_host: "simon@doorcome.cn"
upload_path: "/var/www/html/echart/research"
# ── 同步 (rsync) ─────────────────────────────────────
# 海外服务器已弃用，以下保留作为参考
sync:
port: 22
sentinel: "/tmp/en_news_sync_done"
pack_dir: "/tmp"
# ── 定时调度 ────────────────────────────────────────
# 全流程：M1 抓取 → M2→M6 管道 → 日报
# 每天 06:00 / 12:00 / 18:00 / 22:00 各执行一次
schedule:
day_cutoff_hour: 6
times: ["06:00", "12:00", "18:00", "22:00"]
# ── 日志 ────────────────────────────────────────────
logging:
level: "INFO"
dir: "logs"