初始化
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
# =============================================
|
||||
# 被其他脚本 source，从 system.yaml 导出环境变量
|
||||
# =============================================
|
||||
# 用法：source scripts/_load_config.sh
|
||||
# =============================================
|
||||
|
||||
_load_yaml_value() {
|
||||
# 从 system.yaml 读取一个值
|
||||
# 用法: _load_yaml_value "servers.overseas_host"
|
||||
local key_path="$1"
|
||||
local default="${2:-}"
|
||||
|
||||
if command -v python3 &>/dev/null; then
|
||||
local value
|
||||
value=$(python3 -c "
|
||||
import yaml, sys
|
||||
try:
|
||||
with open('configs/system.yaml') as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
parts = '${key_path}'.split('.')
|
||||
val = cfg
|
||||
for p in parts:
|
||||
val = val[p]
|
||||
print(val)
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
" 2>/dev/null) && echo "$value" && return
|
||||
fi
|
||||
echo "$default"
|
||||
}
|
||||
|
||||
# ── 导出脚本需要的变量 ──────────────────────────
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
CONFIG_DIR="$SCRIPT_DIR/../configs"
|
||||
|
||||
if [ -f "$CONFIG_DIR/system.yaml" ]; then
|
||||
export CRAWL_PROJECT_DIR="$(_load_yaml_value "servers.overseas_path" "/opt/intlgrab")"
|
||||
export LOCAL_PROJECT_DIR="$(_load_yaml_value "servers.domestic_path" "/home/pi/intlnews")"
|
||||
export OVERSEAS_HOST="$(_load_yaml_value "servers.overseas_host" "")"
|
||||
export SYNC_SENTINEL="$(_load_yaml_value "sync.sentinel" "/tmp/en_news_sync_done")"
|
||||
export SYNC_PACK_DIR="$(_load_yaml_value "sync.pack_dir" "/tmp")"
|
||||
else
|
||||
echo "WARNING: configs/system.yaml not found, using defaults"
|
||||
export CRAWL_PROJECT_DIR="/opt/intlgrab"
|
||||
export LOCAL_PROJECT_DIR="/home/pi/intlnews"
|
||||
fi
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# 日志清理：删除 14 天前的日志文件
|
||||
# 适用于海外 + 国内服务器
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
LOG_DIR="$PROJECT_DIR/logs"
|
||||
RETENTION_DAYS=14
|
||||
|
||||
if [ ! -d "$LOG_DIR" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
DELETED=$(find "$LOG_DIR" -type f -name "*.log" -mtime +$RETENTION_DAYS 2>/dev/null | wc -l)
|
||||
|
||||
if [ "$DELETED" -gt 0 ]; then
|
||||
find "$LOG_DIR" -type f -name "*.log" -mtime +$RETENTION_DAYS -delete 2>/dev/null
|
||||
echo "[$(date)] 清理完成: 删除 $DELETED 个旧日志 (>${RETENTION_DAYS}d)"
|
||||
else
|
||||
echo "[$(date)] 无旧日志需要清理"
|
||||
fi
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# 国内服务器：2G headless stealth 抓取
|
||||
# =============================================
|
||||
# 适用场景：
|
||||
# - 日常轻量补充抓取
|
||||
# - RSS 优先 + headless stealth Web 回退
|
||||
# - 可选 SOCKS5 代理（修改 profiles/2g_headless.yaml 中 proxy.enabled）
|
||||
#
|
||||
# 用法：
|
||||
# bash scripts/domestic_crawl_2g.sh # 抓取全部源
|
||||
# bash scripts/domestic_crawl_2g.sh reuters # 只抓取指定源
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
LOG() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||
|
||||
LOG "══════ 国内 2G headless stealth 抓取 ══════"
|
||||
LOG "Profile: 2g_headless"
|
||||
LOG "内存上限: 1800 MB"
|
||||
LOG "浏览器模式: stealth headless"
|
||||
|
||||
# ── 加载 .env ──
|
||||
export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true)
|
||||
|
||||
# ── 设置 Profile ──
|
||||
export EN_NEWS_PROFILE="2g_headless"
|
||||
|
||||
# ── 执行抓取 ──
|
||||
SOURCE_ARG=""
|
||||
if [ $# -ge 1 ]; then
|
||||
SOURCE_ARG="--source $1"
|
||||
LOG "目标源: $1"
|
||||
else
|
||||
LOG "目标源: 全部启用源"
|
||||
fi
|
||||
|
||||
PYTHONPATH=. .venv/bin/python3 -c "
|
||||
from crawler.orchestrator import run_crawl_sync
|
||||
import sys
|
||||
source_filter = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
stats = run_crawl_sync(source_filter=source_filter)
|
||||
print(f'抓取完成: {stats.sources_crawled} 源, {stats.total_articles} 篇')
|
||||
" "$1"
|
||||
|
||||
LOG "══════ 2G 抓取完成 ✅ ══════"
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# 国内服务器：8G headful + HTTP 代理抓取
|
||||
# =============================================
|
||||
# 适用场景：
|
||||
# - 所有新闻源直接使用 headful Playwright 浏览器抓取
|
||||
# - HTTP 代理 (127.0.0.1:3128) → Privoxy → SOCKS5(ss-local) 访问海外
|
||||
# - 比海外 RSS 方式抓取更可靠，解决反爬问题
|
||||
#
|
||||
# 前置条件：
|
||||
# sudo apt install xvfb
|
||||
# Xvfb :99 -screen 0 1280x1024x24 & # 首次启动
|
||||
# HTTP 代理 127.0.0.1:3128 已运行（privoxy）
|
||||
#
|
||||
# 用法：
|
||||
# bash scripts/domestic_crawl_8g.sh # 抓取全部源
|
||||
# bash scripts/domestic_crawl_8g.sh reuters # 只抓取指定源
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
LOG() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||
|
||||
LOG "══════ 国内 8G headful + HTTP 代理抓取 ══════"
|
||||
LOG "Profile: 8g_headful"
|
||||
LOG "内存上限: 7500 MB"
|
||||
LOG "浏览器模式: headful (Xvfb :99)"
|
||||
LOG "HTTP 代理: 127.0.0.1:3128 (Privoxy → SOCKS5)"
|
||||
|
||||
# ── 加载 .env ──
|
||||
export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true)
|
||||
|
||||
# ── 启动 Xvfb（如果尚未运行）──
|
||||
if ! pgrep -x "Xvfb" > /dev/null; then
|
||||
LOG "启动 Xvfb 虚拟显示器 :99 ..."
|
||||
Xvfb :99 -screen 0 1280x1024x24 -ac +extension RANDR &
|
||||
sleep 1
|
||||
LOG "Xvfb 已启动 (PID: $(pgrep -x Xvfb))"
|
||||
else
|
||||
LOG "Xvfb 已在运行 (PID: $(pgrep -x Xvfb))"
|
||||
fi
|
||||
|
||||
export DISPLAY=:99
|
||||
|
||||
# ── 设置 Profile ──
|
||||
export EN_NEWS_PROFILE="8g_headful"
|
||||
|
||||
# ── 执行抓取 ──
|
||||
SOURCE_ARG=""
|
||||
if [ $# -ge 1 ]; then
|
||||
SOURCE_ARG="--source $1"
|
||||
LOG "目标源: $1"
|
||||
else
|
||||
LOG "目标源: 全部启用源"
|
||||
fi
|
||||
|
||||
PYTHONPATH=. .venv/bin/python3 -c "
|
||||
from crawler.orchestrator import run_crawl_sync
|
||||
import sys
|
||||
source_filter = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
stats = run_crawl_sync(source_filter=source_filter)
|
||||
print(f'抓取完成: {stats.sources_crawled} 源, {stats.total_articles} 篇')
|
||||
" "${1:-}"
|
||||
|
||||
LOG "══════ 8G 抓取完成 ✅ ══════"
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# 国内服务器：全流程自动化（独立运行，无需海外）
|
||||
# 1. M1 Pi 抓取（headful Playwright + HTTP 代理）
|
||||
# 2. 全链路 M2→M6（含日报）
|
||||
# =============================================
|
||||
# 每天 06:00 / 12:00 / 18:00 / 22:00 各执行一次
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
LOG() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||
|
||||
LOG "══════ 国内全流程开始 ══════"
|
||||
|
||||
# ── 加载 .env ──
|
||||
export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true)
|
||||
|
||||
# ── 1. M1 Pi 抓取（headful Playwright + HTTP 代理）──
|
||||
LOG "[1/2] Pi M1 抓取（8G headful + HTTP 代理）..."
|
||||
bash "$SCRIPT_DIR/domestic_crawl_8g.sh" 2>&1 | tail -5 || LOG "WARNING: 部分源抓取失败，继续管道"
|
||||
|
||||
# ── 2. M2→M6 管道（含日报）──
|
||||
LOG "[2/2] 全链路管道..."
|
||||
bash "$SCRIPT_DIR/pipeline.sh" 2>&1 | tail -10
|
||||
|
||||
LOG "══════ 国内全流程完成 ✅ ══════"
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# 国内服务器：从海外拉取数据并合并到本地 data/raw/
|
||||
# =============================================
|
||||
# 幂等：多次执行不会产生重复数据
|
||||
# 用法：./scripts/domestic_sync.sh [日期]
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
# 从 system.yaml 读取配置
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/_load_config.sh"
|
||||
|
||||
LOCAL_DATA="$LOCAL_PROJECT_DIR/data/raw"
|
||||
OVERSEAS_RAW="$CRAWL_PROJECT_DIR/data/raw"
|
||||
PACK_TMP="${SYNC_PACK_DIR:-/tmp}"
|
||||
|
||||
# ── 确定新闻日 ──────────────────────────────────────
|
||||
if [ $# -ge 1 ]; then
|
||||
NEWS_DAY="$1"
|
||||
else
|
||||
CUTOFF=6
|
||||
HOUR=$(date +%H)
|
||||
if [ "$HOUR" -lt "$CUTOFF" ]; then
|
||||
NEWS_DAY=$(date -d "yesterday" +%Y%m%d)
|
||||
else
|
||||
NEWS_DAY=$(date +%Y%m%d)
|
||||
fi
|
||||
fi
|
||||
|
||||
PACK_FILE="en_news_${NEWS_DAY}.tar.gz"
|
||||
|
||||
echo "[$(date)] ═══ 国内同步开始，新闻日: $NEWS_DAY ═══"
|
||||
|
||||
# ── 1. 增量 rsync ─────────────────────────────────
|
||||
echo "[$(date)] [1/4] rsync 增量同步 data/raw/ ..."
|
||||
mkdir -p "$LOCAL_DATA"
|
||||
rsync -avz --ignore-existing \
|
||||
-e "ssh -o ConnectTimeout=10" \
|
||||
"$OVERSEAS_HOST:$OVERSEAS_RAW/" \
|
||||
"$LOCAL_DATA/" \
|
||||
2>&1 | tail -3
|
||||
|
||||
# ── 2. 拉取压缩包 ─────────────────────────────────
|
||||
echo "[$(date)] [2/4] 尝试拉取压缩包 ..."
|
||||
PACK_LOCAL="$PACK_TMP/$PACK_FILE"
|
||||
|
||||
if ssh -o ConnectTimeout=5 "$OVERSEAS_HOST" "[ -f $OVERSEAS_RAW/$PACK_FILE ]" 2>/dev/null; then
|
||||
scp "$OVERSEAS_HOST:$OVERSEAS_RAW/$PACK_FILE" "$PACK_LOCAL" 2>/dev/null
|
||||
echo "[$(date)] 压缩包拉取成功: $PACK_LOCAL ($(du -h "$PACK_LOCAL" | cut -f1))"
|
||||
elif ssh -o ConnectTimeout=5 "$OVERSEAS_HOST" "[ -f /tmp/$PACK_FILE ]" 2>/dev/null; then
|
||||
scp "$OVERSEAS_HOST:/tmp/$PACK_FILE" "$PACK_LOCAL" 2>/dev/null
|
||||
echo "[$(date)] 压缩包拉取成功 (海外 /tmp): $PACK_LOCAL ($(du -h "$PACK_LOCAL" | cut -f1))"
|
||||
else
|
||||
echo "[$(date)] 海外无压缩包，跳过"
|
||||
fi
|
||||
|
||||
# ── 3. 解压合并 ───────────────────────────────────
|
||||
if [ -f "$PACK_LOCAL" ]; then
|
||||
echo "[$(date)] [3/4] 解压合并到 $LOCAL_DATA ..."
|
||||
tar xzf "$PACK_LOCAL" -C "$LOCAL_DATA/" --overwrite 2>&1
|
||||
echo "[$(date)] 解压完成"
|
||||
mkdir -p "$LOCAL_PROJECT_DIR/data/archive"
|
||||
mv "$PACK_LOCAL" "$LOCAL_PROJECT_DIR/data/archive/$PACK_FILE" 2>/dev/null || true
|
||||
else
|
||||
echo "[$(date)] [3/4] 无压缩包，跳过解压"
|
||||
fi
|
||||
|
||||
# ── 4. 验证 ───────────────────────────────────────
|
||||
echo "[$(date)] [4/4] 验证..."
|
||||
echo "[$(date)] 文件数: $(find "$LOCAL_DATA" -type f ! -name '*.tar.gz' | wc -l)"
|
||||
echo "[$(date)] 总大小: $(du -sh "$LOCAL_DATA" 2>/dev/null | cut -f1)"
|
||||
echo "[$(date)] ═══ 国内同步完成 ✅ ═══"
|
||||
Executable
+183
@@ -0,0 +1,183 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# 精准修复: 为 publish_time 为空的存量数据补日期
|
||||
# =============================================
|
||||
# 场景: extractor 升级后，已处理的文章 publish_time 仍为空
|
||||
# 策略: 重新提取日期 → 更新 processed JSON → 更新 events JSON
|
||||
# 影响: 仅修改 JSON 文件的 publish_time 字段，不触发 LLM 重跑
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
echo "[$(date)] ═══ publish_time 精准修复开始 ═══"
|
||||
|
||||
.venv/bin/python3 << 'PYEOF'
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(levelname)s - %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 复用 extractor 的日期提取（不依赖 HTML 文件存在） ──
|
||||
|
||||
def extract_date_from_url(url: str) -> str:
|
||||
"""从 URL 提取日期。"""
|
||||
import re
|
||||
from datetime import datetime
|
||||
url_patterns = [
|
||||
r"/(\d{4})/(\d{2})/(\d{2})/",
|
||||
r"/(\d{4})(\d{2})(\d{2})/",
|
||||
r"-(\d{4})(\d{2})(\d{2})(?:[/-]|$)",
|
||||
r"-(\d{4})-(\d{2})-(\d{2})[/-]",
|
||||
]
|
||||
for pat in url_patterns:
|
||||
m = re.search(pat, url)
|
||||
if m:
|
||||
try:
|
||||
y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
|
||||
return datetime(y, mo, d).isoformat()
|
||||
except ValueError:
|
||||
continue
|
||||
return ""
|
||||
|
||||
|
||||
def extract_date_from_html(html_path: str, url: str) -> str:
|
||||
"""从 HTML 文件提取日期。"""
|
||||
from extractor.extractor import _extract_publish_time
|
||||
p = Path(html_path)
|
||||
if p.exists():
|
||||
try:
|
||||
html = p.read_text(encoding="utf-8")
|
||||
return _extract_publish_time(html, url=url) or ""
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
def extract_date_from_md(md_path: str, url: str) -> str:
|
||||
"""从 Markdown 文件提取日期（RSS 源在 MD 中写入发布时间）。"""
|
||||
import re
|
||||
p = Path(md_path)
|
||||
if not p.exists():
|
||||
return ""
|
||||
try:
|
||||
md = p.read_text(encoding="utf-8")
|
||||
# RSS 格式: **发布时间**: 2026-06-19T14:30:00
|
||||
m = re.search(r"\*\*发布时间\*\*:?\s*([^\n]+)", md)
|
||||
if m:
|
||||
return m.group(1).strip()
|
||||
except Exception:
|
||||
pass
|
||||
return extract_date_from_url(url)
|
||||
|
||||
|
||||
# ════════════════════════════════════════════
|
||||
# 阶段 1: 修复 data/processed/ 中的 publish_time
|
||||
# ════════════════════════════════════════════
|
||||
|
||||
logger.info("═══ 阶段 1: 扫描 data/processed/ ═══")
|
||||
fixed_processed = 0
|
||||
skipped_ok = 0
|
||||
skipped_no_source = 0
|
||||
|
||||
for proc_file in Path("data/processed").glob("*/*/*.json"):
|
||||
if proc_file.name == "index.jsonl":
|
||||
continue
|
||||
try:
|
||||
data = json.loads(proc_file.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# 只处理 publish_time 为空的
|
||||
pt = (data.get("publish_time") or "").strip()
|
||||
if pt:
|
||||
skipped_ok += 1
|
||||
continue
|
||||
|
||||
url = data.get("url", "")
|
||||
html_path = data.get("html_path", "")
|
||||
md_path = data.get("md_path", "")
|
||||
|
||||
# 尝试提取日期
|
||||
new_pt = ""
|
||||
if html_path:
|
||||
new_pt = extract_date_from_html(html_path, url)
|
||||
if not new_pt and md_path:
|
||||
new_pt = extract_date_from_md(md_path, url)
|
||||
if not new_pt:
|
||||
new_pt = extract_date_from_url(url)
|
||||
|
||||
if new_pt:
|
||||
data["publish_time"] = new_pt
|
||||
proc_file.write_text(
|
||||
json.dumps(data, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
fixed_processed += 1
|
||||
logger.debug("processed: %s → %s", proc_file.name, new_pt)
|
||||
else:
|
||||
skipped_no_source += 1
|
||||
|
||||
logger.info(
|
||||
"阶段 1 完成: 修复 %d, 已有日期 %d, 仍无法提取 %d",
|
||||
fixed_processed, skipped_ok, skipped_no_source,
|
||||
)
|
||||
|
||||
|
||||
# ════════════════════════════════════════════
|
||||
# 阶段 2: 同步修复 data/events/ 中的 publish_time
|
||||
# ════════════════════════════════════════════
|
||||
|
||||
logger.info("═══ 阶段 2: 扫描 data/events/ ═══")
|
||||
fixed_events = 0
|
||||
skipped_events_ok = 0
|
||||
skipped_events_no_url = 0
|
||||
|
||||
for ev_file in Path("data/events").glob("*/*.json"):
|
||||
if ev_file.name == "index.json":
|
||||
continue
|
||||
try:
|
||||
data = json.loads(ev_file.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
pt = (data.get("publish_time") or "").strip()
|
||||
if pt:
|
||||
skipped_events_ok += 1
|
||||
continue
|
||||
|
||||
url = data.get("url", "")
|
||||
new_pt = extract_date_from_url(url)
|
||||
|
||||
if new_pt:
|
||||
data["publish_time"] = new_pt
|
||||
ev_file.write_text(
|
||||
json.dumps(data, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
fixed_events += 1
|
||||
logger.debug("events: %s → %s", ev_file.name, new_pt)
|
||||
else:
|
||||
skipped_events_no_url += 1
|
||||
|
||||
logger.info(
|
||||
"阶段 2 完成: 修复 %d, 已有日期 %d, 仍无法提取 %d",
|
||||
fixed_events, skipped_events_ok, skipped_events_no_url,
|
||||
)
|
||||
|
||||
# ── 汇总 ──
|
||||
total_fixed = fixed_processed + fixed_events
|
||||
total_still_empty = skipped_no_source + skipped_events_no_url
|
||||
logger.info("══════ 汇总 ══════")
|
||||
logger.info("processes 修复: %d, events 修复: %d", fixed_processed, fixed_events)
|
||||
logger.info("总计修复: %d, 仍为空: %d", total_fixed, total_still_empty)
|
||||
PYEOF
|
||||
|
||||
echo "[$(date)] ═══ publish_time 精准修复完成 ✅ ═══"
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# 海外服务器：抓取英文财经新闻
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
echo "[$(date)] ═══ 海外抓取开始 ═══"
|
||||
|
||||
PYTHONPATH=. .venv/bin/python3 -c "
|
||||
from crawler.orchestrator import run_crawl_sync
|
||||
stats = run_crawl_sync()
|
||||
print(f'抓取完成: {stats.sources_crawled} 源, {stats.total_articles} 篇')
|
||||
"
|
||||
|
||||
echo "[$(date)] ═══ 海外抓取完成 ✅ ═══"
|
||||
@@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# 海外服务器：打包压缩当日 data/raw/ 下的所有文件
|
||||
# =============================================
|
||||
# 用法：./scripts/overseas_pack.sh [日期]
|
||||
# 不传日期则使用当前新闻日（基于 day_cutoff_hour）
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
# 从 system.yaml 读取配置（有默认值兜底）
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/_load_config.sh"
|
||||
|
||||
PROJECT_DIR="${CRAWL_PROJECT_DIR}"
|
||||
DATA_DIR="$PROJECT_DIR/data/raw"
|
||||
PACK_TMP="${SYNC_PACK_DIR:-/tmp}"
|
||||
SENTINEL_FILE="${SYNC_SENTINEL}"
|
||||
|
||||
# ── 确定新闻日 ──────────────────────────────────────
|
||||
if [ $# -ge 1 ]; then
|
||||
NEWS_DAY="$1"
|
||||
else
|
||||
# 从 system.yaml 读取 cutoff hour
|
||||
CUTOFF=$(python3 -c "
|
||||
import yaml
|
||||
with open('$PROJECT_DIR/configs/system.yaml') as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
print(cfg.get('schedule', {}).get('day_cutoff_hour', 6))
|
||||
" 2>/dev/null || echo 6)
|
||||
|
||||
HOUR=$(date +%H)
|
||||
if [ "$HOUR" -lt "$CUTOFF" ]; then
|
||||
NEWS_DAY=$(date -d "yesterday" +%Y%m%d)
|
||||
else
|
||||
NEWS_DAY=$(date +%Y%m%d)
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[$(date)] 新闻日: $NEWS_DAY"
|
||||
|
||||
# ── 检查数据是否存在 ─────────────────────────────────
|
||||
if [ ! -d "$DATA_DIR" ]; then
|
||||
echo "[$(date)] WARNING: data/raw/ 不存在，跳过打包"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 计算今天目录下的总文件数（排除 tar.gz 和 sentinel）
|
||||
TOTAL_FILES=$(find "$DATA_DIR" -path "*/$NEWS_DAY/*" -type f ! -name "*.tar.gz" 2>/dev/null | wc -l)
|
||||
if [ "$TOTAL_FILES" -eq 0 ]; then
|
||||
echo "[$(date)] WARNING: 新闻日 $NEWS_DAY 无数据文件，跳过打包"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── 打包压缩 ─────────────────────────────────────────
|
||||
PACK_FILE="$PACK_TMP/en_news_${NEWS_DAY}.tar.gz"
|
||||
|
||||
echo "[$(date)] 打包 $TOTAL_FILES 个文件 → $PACK_FILE ..."
|
||||
|
||||
# 收集当天的所有数据文件（包括任意源下面的日期目录）
|
||||
find "$DATA_DIR" -path "*/$NEWS_DAY/*" -type f ! -name "*.tar.gz" 2>/dev/null \
|
||||
| tar czf "$PACK_FILE" -T - --transform='s|.*/data/raw/||' 2>/dev/null
|
||||
|
||||
PACK_SIZE=$(du -h "$PACK_FILE" | cut -f1)
|
||||
echo "[$(date)] 打包完成: $PACK_FILE ($PACK_SIZE)"
|
||||
|
||||
# ── 生成哨兵文件 ─────────────────────────────────────
|
||||
echo "$NEWS_DAY $(date -Iseconds) $PACK_SIZE ($TOTAL_FILES files)" > "$SENTINEL_FILE"
|
||||
echo "[$(date)] 哨兵文件已更新: $SENTINEL_FILE"
|
||||
|
||||
echo "[$(date)] ✅ 海外打包完成"
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# 国内服务器：全链路管道 M2 → M3 → M4 → M5 → M6 → 日报
|
||||
# =============================================
|
||||
# 用法：./scripts/pipeline.sh
|
||||
# 前提：domestic_sync.sh 已完成
|
||||
# =============================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
echo "[$(date)] ═══ 全链路管道开始 ═══"
|
||||
|
||||
# 加载 .env
|
||||
export $(grep -v '^#' .env | grep -v '^$' | xargs 2>/dev/null || true)
|
||||
|
||||
# ── M2: 正文提取 ──
|
||||
echo "[$(date)] [M2] 正文提取..."
|
||||
.venv/bin/python3 -c "
|
||||
from extractor.pipeline import process_all_sources
|
||||
stats = process_all_sources()
|
||||
print(f'M2: {stats[\"total_articles\"]} 篇, {stats[\"elapsed_sec\"]:.0f}s')
|
||||
"
|
||||
|
||||
# ── M3: 去重 ──
|
||||
echo "[$(date)] [M3] 三层去重..."
|
||||
.venv/bin/python3 -c "
|
||||
from dedup.pipeline import dedup_all_sources
|
||||
stats = dedup_all_sources()
|
||||
print(f'M3: 唯一 {stats[\"unique\"]}/重复 {stats[\"duplicate\"]}, {stats[\"elapsed_sec\"]:.0f}s')
|
||||
"
|
||||
|
||||
# ── M4: 翻译+事件 ──
|
||||
echo "[$(date)] [M4] 翻译+事件抽取..."
|
||||
.venv/bin/python3 -c "
|
||||
from llm.pipeline import translate_all_deduped
|
||||
stats = translate_all_deduped()
|
||||
print(f'M4: {stats[\"success\"]}/{stats[\"total\"]} 篇, {stats[\"elapsed_sec\"]:.0f}s')
|
||||
"
|
||||
|
||||
# ── M5: 向量生成 ──
|
||||
echo "[$(date)] [M5] 向量生成..."
|
||||
.venv/bin/python3 -c "
|
||||
from embedding.pipeline import embed_all_events
|
||||
stats = embed_all_events()
|
||||
print(f'M5: {stats[\"success\"]}/{stats[\"total\"]} 篇, {stats[\"elapsed_sec\"]:.0f}s')
|
||||
"
|
||||
|
||||
# ── M6: Qdrant 入库 ──
|
||||
echo "[$(date)] [M6] Qdrant 入库..."
|
||||
.venv/bin/python3 -c "
|
||||
from vectorstore.pipeline import ingest_all_embeddings
|
||||
stats = ingest_all_embeddings()
|
||||
print(f'M6: {stats[\"ingested\"]}/{stats[\"total\"]} 条, {stats[\"elapsed_sec\"]:.0f}s')
|
||||
"
|
||||
|
||||
# ── 日报 ──
|
||||
echo "[$(date)] [日报] 生成日报..."
|
||||
.venv/bin/python3 -c "
|
||||
from scheduler.reporter import generate_report
|
||||
path = generate_report()
|
||||
print(f'日报: {path}')
|
||||
"
|
||||
|
||||
echo "[$(date)] ═══ 全链路管道完成 ✅ ═══"
|
||||
Reference in New Issue
Block a user