初始化
This commit is contained in:
+238
@@ -0,0 +1,238 @@
|
||||
"""批量翻译 + 事件抽取管道。
|
||||
|
||||
输入: data/deduped/{YYYYMMDD}/uniques/{url_hash}.json（M3 去重后唯一条目）
|
||||
输出: data/events/{YYYYMMDD}/{url_hash}.json
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from crawler.utils import get_news_day
|
||||
from extractor.models import ProcessedArticle
|
||||
from llm.client import LLMConfig, load_llm_config, make_sync_client
|
||||
from llm.extractor import PromptTemplate, translate_and_extract
|
||||
from llm.models import EnTranslatedArticle, LLMCallError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 默认并发数
|
||||
_DEFAULT_CONCURRENCY = 3
|
||||
|
||||
|
||||
def _load_concurrency() -> int:
|
||||
"""从 system.yaml 读取 LLM 并发配置。"""
|
||||
config_path = Path("configs/system.yaml")
|
||||
if config_path.exists():
|
||||
try:
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
raw = yaml.safe_load(f)
|
||||
return int(raw.get("llm", {}).get("concurrency", _DEFAULT_CONCURRENCY))
|
||||
except Exception:
|
||||
pass
|
||||
return _DEFAULT_CONCURRENCY
|
||||
|
||||
|
||||
def _load_deduped_articles(
|
||||
date_str: str,
|
||||
) -> list[ProcessedArticle]:
|
||||
"""加载指定日期的去重后唯一条目。
|
||||
|
||||
Args:
|
||||
date_str: 日期 YYYYMMDD
|
||||
|
||||
Returns:
|
||||
ProcessedArticle 列表
|
||||
"""
|
||||
base_dir = Path(f"data/deduped/{date_str}/uniques")
|
||||
if not base_dir.exists():
|
||||
return []
|
||||
|
||||
articles: list[ProcessedArticle] = []
|
||||
for json_file in sorted(base_dir.glob("*.json")):
|
||||
try:
|
||||
data = json.loads(json_file.read_text(encoding="utf-8"))
|
||||
articles.append(ProcessedArticle(**data))
|
||||
except (json.JSONDecodeError, Exception) as e:
|
||||
logger.warning("解析去重文章失败 %s: %s", json_file, e)
|
||||
|
||||
return articles
|
||||
|
||||
|
||||
def _process_one(
|
||||
article: ProcessedArticle,
|
||||
client,
|
||||
config: LLMConfig,
|
||||
template: PromptTemplate,
|
||||
) -> EnTranslatedArticle | None:
|
||||
"""处理单篇文章的翻译 + 事件抽取。返回 None 表示失败。"""
|
||||
try:
|
||||
return translate_and_extract(
|
||||
client=client,
|
||||
config=config,
|
||||
article=article,
|
||||
template=template,
|
||||
)
|
||||
except LLMCallError as e:
|
||||
logger.error(
|
||||
"翻译抽取最终失败 url=%s: %s", article.url, e.reason
|
||||
)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.exception("翻译抽取未预期异常 url=%s: %s", article.url, e)
|
||||
return None
|
||||
|
||||
|
||||
def translate_all_deduped(
|
||||
date_str: str | None = None,
|
||||
*,
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
concurrency: int | None = None,
|
||||
) -> dict:
|
||||
"""对去重后的所有唯一条目执行翻译 + 事件抽取。
|
||||
|
||||
Args:
|
||||
date_str: 日期 YYYYMMDD，默认当前新闻日
|
||||
provider: LLM provider，默认从 system.yaml 读取
|
||||
model: LLM model，默认从 system.yaml 读取
|
||||
concurrency: 并发数，默认从 system.yaml 读取
|
||||
|
||||
Returns:
|
||||
统计 dict
|
||||
"""
|
||||
if date_str is None:
|
||||
date_str = get_news_day()
|
||||
|
||||
concurrency = concurrency or _load_concurrency()
|
||||
|
||||
logger.info("══════ 开始翻译+事件抽取，日期: %s，并发: %d ══════",
|
||||
date_str, concurrency)
|
||||
|
||||
# 加载去重后的文章
|
||||
articles = _load_deduped_articles(date_str)
|
||||
if not articles:
|
||||
logger.warning("去重目录无文章: data/deduped/%s/uniques/", date_str)
|
||||
return {"date": date_str, "total": 0, "success": 0, "failed": 0, "elapsed_sec": 0}
|
||||
|
||||
# 初始化 LLM 客户端
|
||||
config = load_llm_config(provider=provider, model=model)
|
||||
client = make_sync_client(config)
|
||||
template = PromptTemplate()
|
||||
|
||||
# 输出目录
|
||||
out_dir = Path(f"data/events/{date_str}")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
start_time = datetime.now()
|
||||
success = 0
|
||||
failed = 0
|
||||
|
||||
# 增量：跳过已翻译的文章
|
||||
new_articles = []
|
||||
skipped = 0
|
||||
for a in articles:
|
||||
if (out_dir / f"{a.url_hash}.json").exists():
|
||||
skipped += 1
|
||||
else:
|
||||
new_articles.append(a)
|
||||
if skipped > 0:
|
||||
logger.info("增量跳过 %d 篇已翻译，剩余 %d 篇待处理", skipped, len(new_articles))
|
||||
articles = new_articles
|
||||
|
||||
# 并发处理
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||||
futures = {
|
||||
executor.submit(_process_one, article, client, config, template): article
|
||||
for article in articles
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
article = futures[future]
|
||||
try:
|
||||
result = future.result()
|
||||
except Exception as e:
|
||||
logger.error("并发任务异常 url=%s: %s", article.url, e)
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
if result is None:
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
# 写入输出文件
|
||||
out_file = out_dir / f"{result.url_hash}.json"
|
||||
out_file.write_text(
|
||||
result.model_dump_json(indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
success += 1
|
||||
logger.info(
|
||||
"[%s] ✅ %s → %s (%d events, %d zh chars)",
|
||||
result.source_id,
|
||||
result.title[:40],
|
||||
result.title_zh[:30],
|
||||
len(result.events),
|
||||
result.word_count_zh,
|
||||
)
|
||||
|
||||
elapsed = (datetime.now() - start_time).total_seconds()
|
||||
|
||||
# 写入事件索引
|
||||
_write_event_index(date_str, success, failed, elapsed, config)
|
||||
|
||||
logger.info(
|
||||
"══════ 翻译+事件抽取完成: 成功 %d / 失败 %d / 总计 %d，耗时 %.1f 秒 ══════",
|
||||
success, failed, len(articles), elapsed,
|
||||
)
|
||||
|
||||
return {
|
||||
"date": date_str,
|
||||
"total": len(articles),
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"elapsed_sec": elapsed,
|
||||
"provider": config.provider,
|
||||
"model": config.model,
|
||||
}
|
||||
|
||||
|
||||
def _write_event_index(
|
||||
date_str: str,
|
||||
success: int,
|
||||
failed: int,
|
||||
elapsed_sec: float,
|
||||
config: LLMConfig,
|
||||
) -> None:
|
||||
"""写入事件索引文件。
|
||||
|
||||
Args:
|
||||
date_str: 日期
|
||||
success: 成功数
|
||||
failed: 失败数
|
||||
elapsed_sec: 耗时（秒）
|
||||
config: LLM 配置
|
||||
"""
|
||||
out_dir = Path(f"data/events/{date_str}")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
index_data = {
|
||||
"date": date_str,
|
||||
"success": success,
|
||||
"failed": failed,
|
||||
"elapsed_sec": round(elapsed_sec, 1),
|
||||
"provider": config.provider,
|
||||
"model": config.model,
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
}
|
||||
|
||||
index_path = out_dir / "index.json"
|
||||
index_path.write_text(
|
||||
json.dumps(index_data, indent=2, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
logger.info("事件索引已写入: %s", index_path)
|
||||
Reference in New Issue
Block a user