Initial commit
This commit is contained in:
@@ -0,0 +1,218 @@
|
||||
# A 股新闻源配置(M1)
|
||||
# ============================================================
|
||||
# 文档:
|
||||
# - id: 短标识,用作目录名(小写字母/数字/下划线)
|
||||
# - name: 中文名,用于日志与展示
|
||||
# - homepage: 列表/首页 URL,抓取入口
|
||||
# - article_url_pattern: 正则,从首页所有链接中筛选文章 URL
|
||||
# - article_link_selector: 可选 CSS,优先在该选择器内查链接(留空=全页)
|
||||
# - js_render: 是否需要 JS 渲染(动态站点必须 true)
|
||||
# - wait_for: 可选 CSS,等待元素出现(JS 渲染时建议设置)
|
||||
# - max_articles_per_run: 单次抓取上限,M1 阶段建议 ≤ 30
|
||||
# ============================================================
|
||||
|
||||
settings:
|
||||
concurrency: 3 # 并发抓取上限(用户决策:3)
|
||||
retry_max_attempts: 3
|
||||
retry_min_wait_sec: 1.0
|
||||
retry_max_wait_sec: 10.0
|
||||
headless: true
|
||||
output_root: data/raw
|
||||
|
||||
sources:
|
||||
# ---- 1. 财联社(深度频道)----
|
||||
- id: cls
|
||||
name: 财联社
|
||||
enabled: true
|
||||
homepage: https://www.cls.cn/depth?id=1000
|
||||
article_url_pattern: '^https?://www\.cls\.cn/detail/\d+$'
|
||||
article_link_selector: null
|
||||
js_render: true
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://www.cls.cn/depth?id=1003
|
||||
- https://www.cls.cn/depth?id=1007
|
||||
- https://www.cls.cn/depth?id=1035
|
||||
- https://www.cls.cn/depth?id=1032
|
||||
- https://www.cls.cn/depth?id=1005
|
||||
|
||||
# ---- 2. 东方财富(财经频道)----
|
||||
- id: eastmoney
|
||||
name: 东方财富
|
||||
enabled: true
|
||||
homepage: https://finance.eastmoney.com/
|
||||
article_url_pattern: '^https?://finance\.eastmoney\.com/a/\d+\.html$'
|
||||
article_link_selector: null
|
||||
js_render: true
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
|
||||
# ---- 3. 新浪财经 ----
|
||||
- id: sina
|
||||
name: 新浪财经
|
||||
enabled: true
|
||||
homepage: https://finance.sina.com.cn/
|
||||
# 新浪文章 URL 形如 https://finance.sina.com.cn/stock/.../doc-xxx.shtml
|
||||
article_url_pattern: '^https?://(finance|cj)\.sina\.com\.cn/.+/doc-[a-z0-9]+\.s?html$'
|
||||
article_link_selector: null
|
||||
js_render: false
|
||||
wait_for: null
|
||||
page_timeout_ms: 20000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://finance.sina.com.cn/stock/newstock/
|
||||
- https://finance.sina.com.cn/stock/
|
||||
- https://finance.sina.com.cn/forex/
|
||||
- https://finance.sina.com.cn/stock/hkstock/
|
||||
- https://finance.sina.com.cn/stock/usstock/
|
||||
|
||||
# ---- 4. 证券时报 ----
|
||||
- id: stcn
|
||||
name: 证券时报
|
||||
enabled: true
|
||||
homepage: https://www.stcn.com/article/list/yw.html
|
||||
article_url_pattern: '^https?://www\.stcn\.com/article/detail/\d+\.html$'
|
||||
article_link_selector: null
|
||||
js_render: true
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://www.stcn.com/article/list/xw.html
|
||||
- https://www.stcn.com/article/list/finance.html
|
||||
- https://www.stcn.com/article/list/gd.html
|
||||
|
||||
# ---- 5. 第一财经 ----
|
||||
- id: yicai
|
||||
name: 第一财经
|
||||
enabled: true
|
||||
homepage: https://www.yicai.com/news/
|
||||
article_url_pattern: '^https?://www\.yicai\.com/(news|brief)/\d+\.html$'
|
||||
article_link_selector: null
|
||||
js_render: true
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://www.yicai.com/brief/
|
||||
|
||||
# ---- 6. 华尔街见闻 ----
|
||||
- id: wallstreetcn
|
||||
name: 华尔街见闻
|
||||
enabled: true
|
||||
# 注意: 该站为 React SPA,js_render 触发反爬拦截(Crawl4AI 内部错误),
|
||||
# 非 JS 模式下 Crawl4AI 可抓取首页链接但文章正文提取为空。
|
||||
# TODO: 研究通过 wallstreetcn API 或 RSS 获取全文
|
||||
homepage: https://wallstreetcn.com/news/global
|
||||
article_url_pattern: '^https?://wallstreetcn\.com/(articles|livenews)/\d+'
|
||||
js_render: false
|
||||
wait_for: null
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://wallstreetcn.com/live/global
|
||||
|
||||
# ---- 7. 新华网 ----
|
||||
- id: newscn
|
||||
name: 新华网
|
||||
enabled: true
|
||||
homepage: https://www.news.cn/finance/
|
||||
article_url_pattern: '^https?://www\.news\.cn/finance/\d+'
|
||||
js_render: false
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://www.news.cn/money/yw/index.html
|
||||
|
||||
# ---- 8. 中国经济网 ----
|
||||
- id: cecn
|
||||
name: 中国经济网
|
||||
enabled: true
|
||||
homepage: http://finance.ce.cn/
|
||||
article_url_pattern: '^https?://[a-z0-9]+\.ce\.cn/.+/t\d{8}_\d+\.shtml$'
|
||||
js_render: false
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- http://finance.ce.cn/rolling/index.shtml
|
||||
|
||||
# ---- 9. 中新经纬 ----
|
||||
- id: jwview
|
||||
name: 中新经纬
|
||||
enabled: true
|
||||
homepage: https://www.jwview.com/jr.html
|
||||
article_url_pattern: '^https?://www\.jwview\.com/jingwei/html/\d{2}-\d{2}/\d+'
|
||||
js_render: false
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://www.jwview.com/kb.html
|
||||
|
||||
# ---- 10. 中国证券网 ----
|
||||
- id: cnstock
|
||||
name: 中国证券网
|
||||
enabled: true
|
||||
homepage: https://www.cnstock.com/channel/10011
|
||||
article_url_pattern: '^https?://www\.cnstock\.com/commonDetail/\d+'
|
||||
js_render: false
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://www.cnstock.com/fastNews/10004
|
||||
|
||||
# ---- 11. 中证券网 ----
|
||||
- id: cscn
|
||||
name: 中证券网
|
||||
enabled: true
|
||||
homepage: https://www.cs.com.cn/yaowen.html
|
||||
article_url_pattern: '^https?://www\.cs\.com\.cn/xwzx/\d{2}/\d{4}/\d{2}/\d+'
|
||||
js_render: false
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://www.cs.com.cn/gongsi.html
|
||||
|
||||
# ---- 12. 证券日报 ----
|
||||
- id: zqrb
|
||||
name: 证券日报
|
||||
enabled: true
|
||||
homepage: http://www.zqrb.cn/finance/index.html
|
||||
# zqrb URL 包含日期: /gscy/gongsi/2026-06-17/A1781692801973.html
|
||||
# 限制只匹配 2025 年后的文章,避免抓取到 2022 年旧闻
|
||||
article_url_pattern: '^https?://www\.zqrb\.cn/(?:gscy/gongsi|finance)/(?:202[5-9]|203\d)-\d{2}-\d{2}/'
|
||||
js_render: false
|
||||
wait_for: "css:body"
|
||||
page_timeout_ms: 30000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- http://www.zqrb.cn/finance/hongguanjingji/index.html
|
||||
- http://www.zqrb.cn/finance/guojijingji/index.html
|
||||
|
||||
# ---- 13. 经济观察网 ----
|
||||
- id: eeo
|
||||
name: 经济观察网
|
||||
enabled: true
|
||||
homepage: https://www.eeo.com.cn/
|
||||
article_url_pattern: '^https?://www\.eeo\.com\.cn/\d{4}/\d{4}/\d+\.shtml$'
|
||||
js_render: false
|
||||
page_timeout_ms: 60000
|
||||
max_articles_per_run: 20
|
||||
extra_homepages:
|
||||
- https://www.eeo.com.cn/jg/kuaixun/
|
||||
|
||||
# ---- 14. 新闻联播 (API, 不走 Playwright) ----
|
||||
- id: xwlb
|
||||
name: 新闻联播
|
||||
enabled: true
|
||||
homepage: https://api.doorcome.cn/api/xwlbFine/ # dummy, 满足模型校验
|
||||
article_url_pattern: '^xwlb://' # dummy, 不会被 crawler 处理
|
||||
js_render: false
|
||||
max_articles_per_run: 25
|
||||
Reference in New Issue
Block a user