simon/news
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
simon
2026-07-18 15:51:01 +08:00
commit f2c80c5a9c
799 changed files with 133475 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/.gitkeep
View File
configs/sources.yaml
+218
View File
@@ -0,0 +1,218 @@
# A 股新闻源配置(M1)
# ============================================================
# 文档:
# - id: 短标识,用作目录名(小写字母/数字/下划线)
# - name: 中文名,用于日志与展示
# - homepage: 列表/首页 URL,抓取入口
# - article_url_pattern: 正则,从首页所有链接中筛选文章 URL
# - article_link_selector: 可选 CSS,优先在该选择器内查链接(留空=全页)
# - js_render: 是否需要 JS 渲染(动态站点必须 true)
# - wait_for: 可选 CSS,等待元素出现(JS 渲染时建议设置)
# - max_articles_per_run: 单次抓取上限,M1 阶段建议 ≤ 30
# ============================================================
settings:
concurrency: 3 # 并发抓取上限(用户决策:3)
retry_max_attempts: 3
retry_min_wait_sec: 1.0
retry_max_wait_sec: 10.0
headless: true
output_root: data/raw
sources:
# ---- 1. 财联社(深度频道)----
- id: cls
name: 财联社
enabled: true
homepage: https://www.cls.cn/depth?id=1000
article_url_pattern: '^https?://www\.cls\.cn/detail/\d+$'
article_link_selector: null
js_render: true
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- https://www.cls.cn/depth?id=1003
- https://www.cls.cn/depth?id=1007
- https://www.cls.cn/depth?id=1035
- https://www.cls.cn/depth?id=1032
- https://www.cls.cn/depth?id=1005
# ---- 2. 东方财富(财经频道)----
- id: eastmoney
name: 东方财富
enabled: true
homepage: https://finance.eastmoney.com/
article_url_pattern: '^https?://finance\.eastmoney\.com/a/\d+\.html$'
article_link_selector: null
js_render: true
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
# ---- 3. 新浪财经 ----
- id: sina
name: 新浪财经
enabled: true
homepage: https://finance.sina.com.cn/
# 新浪文章 URL 形如 https://finance.sina.com.cn/stock/.../doc-xxx.shtml
article_url_pattern: '^https?://(finance|cj)\.sina\.com\.cn/.+/doc-[a-z0-9]+\.s?html$'
article_link_selector: null
js_render: false
wait_for: null
page_timeout_ms: 20000
max_articles_per_run: 20
extra_homepages:
- https://finance.sina.com.cn/stock/newstock/
- https://finance.sina.com.cn/stock/
- https://finance.sina.com.cn/forex/
- https://finance.sina.com.cn/stock/hkstock/
- https://finance.sina.com.cn/stock/usstock/
# ---- 4. 证券时报 ----
- id: stcn
name: 证券时报
enabled: true
homepage: https://www.stcn.com/article/list/yw.html
article_url_pattern: '^https?://www\.stcn\.com/article/detail/\d+\.html$'
article_link_selector: null
js_render: true
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- https://www.stcn.com/article/list/xw.html
- https://www.stcn.com/article/list/finance.html
- https://www.stcn.com/article/list/gd.html
# ---- 5. 第一财经 ----
- id: yicai
name: 第一财经
enabled: true
homepage: https://www.yicai.com/news/
article_url_pattern: '^https?://www\.yicai\.com/(news|brief)/\d+\.html$'
article_link_selector: null
js_render: true
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- https://www.yicai.com/brief/
# ---- 6. 华尔街见闻 ----
- id: wallstreetcn
name: 华尔街见闻
enabled: true
# 注意: 该站为 React SPA,js_render 触发反爬拦截(Crawl4AI 内部错误),
# 非 JS 模式下 Crawl4AI 可抓取首页链接但文章正文提取为空。
# TODO: 研究通过 wallstreetcn API 或 RSS 获取全文
homepage: https://wallstreetcn.com/news/global
article_url_pattern: '^https?://wallstreetcn\.com/(articles|livenews)/\d+'
js_render: false
wait_for: null
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- https://wallstreetcn.com/live/global
# ---- 7. 新华网 ----
- id: newscn
name: 新华网
enabled: true
homepage: https://www.news.cn/finance/
article_url_pattern: '^https?://www\.news\.cn/finance/\d+'
js_render: false
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- https://www.news.cn/money/yw/index.html
# ---- 8. 中国经济网 ----
- id: cecn
name: 中国经济网
enabled: true
homepage: http://finance.ce.cn/
article_url_pattern: '^https?://[a-z0-9]+\.ce\.cn/.+/t\d{8}_\d+\.shtml$'
js_render: false
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- http://finance.ce.cn/rolling/index.shtml
# ---- 9. 中新经纬 ----
- id: jwview
name: 中新经纬
enabled: true
homepage: https://www.jwview.com/jr.html
article_url_pattern: '^https?://www\.jwview\.com/jingwei/html/\d{2}-\d{2}/\d+'
js_render: false
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- https://www.jwview.com/kb.html
# ---- 10. 中国证券网 ----
- id: cnstock
name: 中国证券网
enabled: true
homepage: https://www.cnstock.com/channel/10011
article_url_pattern: '^https?://www\.cnstock\.com/commonDetail/\d+'
js_render: false
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- https://www.cnstock.com/fastNews/10004
# ---- 11. 中证券网 ----
- id: cscn
name: 中证券网
enabled: true
homepage: https://www.cs.com.cn/yaowen.html
article_url_pattern: '^https?://www\.cs\.com\.cn/xwzx/\d{2}/\d{4}/\d{2}/\d+'
js_render: false
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- https://www.cs.com.cn/gongsi.html
# ---- 12. 证券日报 ----
- id: zqrb
name: 证券日报
enabled: true
homepage: http://www.zqrb.cn/finance/index.html
# zqrb URL 包含日期: /gscy/gongsi/2026-06-17/A1781692801973.html
# 限制只匹配 2025 年后的文章,避免抓取到 2022 年旧闻
article_url_pattern: '^https?://www\.zqrb\.cn/(?:gscy/gongsi|finance)/(?:202[5-9]|203\d)-\d{2}-\d{2}/'
js_render: false
wait_for: "css:body"
page_timeout_ms: 30000
max_articles_per_run: 20
extra_homepages:
- http://www.zqrb.cn/finance/hongguanjingji/index.html
- http://www.zqrb.cn/finance/guojijingji/index.html
# ---- 13. 经济观察网 ----
- id: eeo
name: 经济观察网
enabled: true
homepage: https://www.eeo.com.cn/
article_url_pattern: '^https?://www\.eeo\.com\.cn/\d{4}/\d{4}/\d+\.shtml$'
js_render: false
page_timeout_ms: 60000
max_articles_per_run: 20
extra_homepages:
- https://www.eeo.com.cn/jg/kuaixun/
# ---- 14. 新闻联播 (API, 不走 Playwright) ----
- id: xwlb
name: 新闻联播
enabled: true
homepage: https://api.doorcome.cn/api/xwlbFine/ # dummy, 满足模型校验
article_url_pattern: '^xwlb://' # dummy, 不会被 crawler 处理
js_render: false
max_articles_per_run: 25
configs/watchlist.yaml
+80
View File
@@ -0,0 +1,80 @@
# cninfo 公告关注列表
# code: 6 位股票代码
# name: 公司简称
# note: 可选备注
# orgId: cninfo 机构 ID（从 cninfo 搜索框输入代码后 URL 中获取）
watchlist:
- code: "000792"
name: "盐湖股份"
note: "盐湖提锂"
orgId: "gssz0000792"
- code: "300316"
name: "晶盛机电"
note: ""
orgId: "9900022122"
- code: "688556"
name: "高测股份"
note: ""
orgId: "gfbj0834278"
- code: "688005"
name: "容百科技"
note: ""
orgId: "9900038937"
- code: "300124"
name: "汇川技术"
note: ""
orgId: "9900012527"
- code: "002241"
name: "歌尔股份"
note: ""
orgId: "9900004688"
- code: "600900"
name: "长江电力"
note: ""
orgId: "gssh0600900"
- code: "601126"
name: "四方股份"
note: ""
orgId: "9900016847"
- code: "600089"
name: "特变电工"
note: ""
orgId: "gssh0600089"
- code: "601179"
name: "中国西电"
note: ""
orgId: "9900010350"
- code: "688676"
name: "金盘科技"
note: ""
orgId: "9900044215"
- code: "688469"
name: "芯联集成"
note: ""
orgId: "9900053045"
- code: "002129"
name: "TCL中环"
note: ""
orgId: "9900002703"
- code: "600460"
name: "士兰微"
note: ""
orgId: "gssh0600460"
- code: "601668"
name: "中国建筑"
note: ""
orgId: "9900005970"