From 0fbd6029cb478a27f89d24a2a67acdb8b6866049 Mon Sep 17 00:00:00 2001 From: RYY177 <3346599702@qq.com> Date: Mon, 20 Jul 2026 14:23:23 +0000 Subject: [PATCH] feat(trainer): add Verl GRPO and persistent worker --- api/app/controllers/task.py | 14 +- api/app/services/config/service.py | 5 +- api/app/services/task/service.py | 86 +++- api/app/utils/config/config.py | 24 +- api/examples/bird.yaml | 4 +- examples/codex_home_example/AGENTS.md | 4 + examples/config/starter.yaml | 23 +- examples/scripts/run_trainer.py | 11 - loopai/common/db_tool/runtime.py | 40 +- loopai/common/db_tool/task.py | 67 ++- loopai/common/i18n/i18n_dict.py | 6 +- loopai/common/tracking.py | 125 +++++ loopai/schema/states.py | 184 +++++-- loopai/skills/Trainer/README.md | 102 +++- .../Trainer/nodes/config_generation_node.py | 47 +- .../skills/Trainer/nodes/data_check_node.py | 46 +- .../Trainer/nodes/training_execution_node.py | 194 ++++++-- loopai/skills/Trainer/results.py | 321 ++++++++++-- loopai/skills/Trainer/rewards/__init__.py | 19 + loopai/skills/Trainer/rewards/router.py | 208 ++++++++ loopai/skills/Trainer/runner.py | 216 ++++++-- loopai/skills/Trainer/runtime_config.py | 344 +++++++++++-- .../Trainer/templates/default_config.json | 2 +- .../templates/qwen2.5_sft_single_gpu.yaml | 3 +- .../qwen2_5_coder_bird_full_sft.yaml | 4 +- .../skills/Trainer/templates/verl_grpo.yaml | 66 +++ .../Trainer/templates/verl_grpo_smoke.yaml | 74 +++ loopai/skills/Trainer/trainer_agent.py | 29 +- .../skills/Trainer/utils/config_generator.py | 33 +- .../skills/Trainer/utils/persistent_worker.py | 335 +++++++++++++ .../Trainer/utils/realtime_log_parser.py | 55 +- loopai/skills/Trainer/utils/stream_events.py | 16 +- loopai/skills/Trainer/utils/task_manager.py | 132 ++--- .../skills/Trainer/utils/training_executor.py | 77 +-- .../Trainer/utils/training_log_parser.py | 130 ++++- .../Trainer/utils/verl_config_generator.py | 146 ++++++ .../skills/Trainer/utils/verl_data_checker.py | 434 ++++++++++++++++ loopai/skills/Trainer/utils/verl_exporter.py | 73 +++ loopai/skills/Trainer/utils/verl_launcher.py | 160 ++++++ loopai/skills/Trainer/worker_entry.py | 185 +++++++ skills/Trainer/SKILL.md | 161 ++++-- tests/test_trainer_config_approval.py | 129 ++++- tests/test_trainer_metrics_api.py | 84 ++++ tests/test_trainer_no_swanlab.py | 289 +++++++++++ tests/test_trainer_persistent_worker.py | 274 ++++++++++ tests/test_trainer_task_lifecycle.py | 14 +- tests/test_trainer_verl_integration.py | 469 ++++++++++++++++++ tutorial/docs/guide/cli-tutorial.md | 13 +- tutorial/docs/guide/details/trainer-agent.md | 26 +- ui/package.json | 2 +- .../statusPreview/trainState/index.vue | 33 +- .../statusPreview/trainState/lineChart.vue | 3 +- 52 files changed, 4937 insertions(+), 604 deletions(-) create mode 100644 loopai/common/tracking.py create mode 100644 loopai/skills/Trainer/rewards/__init__.py create mode 100644 loopai/skills/Trainer/rewards/router.py rename loopai/{agents => skills}/Trainer/templates/qwen2.5_sft_single_gpu.yaml (95%) create mode 100644 loopai/skills/Trainer/templates/verl_grpo.yaml create mode 100644 loopai/skills/Trainer/templates/verl_grpo_smoke.yaml create mode 100644 loopai/skills/Trainer/utils/persistent_worker.py create mode 100644 loopai/skills/Trainer/utils/verl_config_generator.py create mode 100644 loopai/skills/Trainer/utils/verl_data_checker.py create mode 100644 loopai/skills/Trainer/utils/verl_exporter.py create mode 100644 loopai/skills/Trainer/utils/verl_launcher.py create mode 100644 loopai/skills/Trainer/worker_entry.py create mode 100644 tests/test_trainer_metrics_api.py create mode 100644 tests/test_trainer_no_swanlab.py create mode 100644 tests/test_trainer_persistent_worker.py create mode 100644 tests/test_trainer_verl_integration.py diff --git a/api/app/controllers/task.py b/api/app/controllers/task.py index 8d919463..7a273d2b 100644 --- a/api/app/controllers/task.py +++ b/api/app/controllers/task.py @@ -1,5 +1,4 @@ import os -import json from fastapi import APIRouter from ..models.body import response_body, TaskItem, TaskStateConfigModel from ..services.task import ( @@ -112,11 +111,8 @@ async def get_latest_runtimes(task_id: str): async def get_train_status(output_dir: str, task_id: str, train_task_id: str): """获取训练状态""" output_dir = _resolve_output_dir(output_dir) - watch_path = os.path.join(output_dir, task_id, 'trainer', train_task_id) - final_path = os.path.join(watch_path, 'metrics', 'metrics.json') - if os.path.exists(final_path): - with open(final_path, 'r') as f: - metrics = json.load(f) - return response_body(data=metrics)() - else: - return response_body(code=404, status='error', message='训练状态文件不存在:' + final_path)() + try: + metrics = get_train_status_service(output_dir, task_id, train_task_id) + return response_body(data=metrics)() + except TaskServiceError as exc: + return response_body(code=exc.code, status='error', message=exc.message)() diff --git a/api/app/services/config/service.py b/api/app/services/config/service.py index 7b4e416d..2c3f6e93 100644 --- a/api/app/services/config/service.py +++ b/api/app/services/config/service.py @@ -8,6 +8,7 @@ from ...models.body import ConfigModel from ...models.db_models import StarterConfig from ...utils.config.config import format_value, get_state_config, get_system_config +from loopai.common.tracking import strip_retired_tracking_fields CURRENT_DIR = Path(__file__).resolve().parent @@ -36,7 +37,7 @@ async def get_starter_config() -> dict[str, Any]: def _parse_config_payload(raw_config: str) -> dict[str, Any]: try: - return json.loads(raw_config) + return strip_retired_tracking_fields(json.loads(raw_config)) except Exception as exc: raise ConfigServiceError("config格式错误") from exc @@ -73,7 +74,7 @@ async def update_starter_config(config: ConfigModel) -> dict[str, Any]: if not original_config: raise ConfigServiceError("config不存在") - original_config_obj = json.loads(original_config.config) + original_config_obj = strip_retired_tracking_fields(json.loads(original_config.config)) _apply_system_config(original_config_obj, config_obj.get("system", {})) _apply_states_config(original_config_obj, config_obj.get("states", {})) diff --git a/api/app/services/task/service.py b/api/app/services/task/service.py index 74c13eee..cded5e7c 100644 --- a/api/app/services/task/service.py +++ b/api/app/services/task/service.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import os import uuid from copy import deepcopy from pathlib import Path @@ -13,6 +12,7 @@ from ...models.db_models import TaskModel, TaskRuntime from ...utils.config.config import get_state_config from ...utils.task.task import apply_state_config_updates, build_task_state_config, config_format +from loopai.common.tracking import strip_retired_tracking_fields CURRENT_DIR = Path(__file__).resolve().parent @@ -28,6 +28,50 @@ def __init__(self, message: str, code: int = 400): self.code = code +def _sanitize_serialized_json(value: Any) -> Any: + """Hide retired tracking secrets from legacy API records.""" + if not isinstance(value, str): + return strip_retired_tracking_fields(value) + try: + parsed = json.loads(value) + except Exception: + return value + return json.dumps(strip_retired_tracking_fields(parsed), ensure_ascii=False) + + +async def _purge_retired_tracking_from_task(task: TaskModel) -> None: + """Remove legacy tracker fields from persisted task config/state once read.""" + changed_fields: list[str] = [] + for field_name in ("config", "state"): + raw_value = getattr(task, field_name, None) + if not isinstance(raw_value, str) or not raw_value: + continue + try: + original = json.loads(raw_value) + except Exception: + continue + cleaned = strip_retired_tracking_fields(original) + if cleaned != original: + setattr(task, field_name, json.dumps(cleaned, ensure_ascii=False)) + changed_fields.append(field_name) + if changed_fields: + await task.save(update_fields=changed_fields) + + +async def _purge_retired_tracking_from_runtime(runtime: TaskRuntime) -> None: + raw_state = runtime.state + if not isinstance(raw_state, str) or not raw_state: + return + try: + original = json.loads(raw_state) + except Exception: + return + cleaned = strip_retired_tracking_fields(original) + if cleaned != original: + runtime.state = json.dumps(cleaned, ensure_ascii=False) + await runtime.save(update_fields=["state"]) + + def _merge_state(base: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]: merged = deepcopy(base) for key, value in overrides.items(): @@ -74,7 +118,7 @@ def _serialize_task_runtime( "updatedAt": runtime.updatedAt, } if include_state: - payload["state"] = runtime.state + payload["state"] = _sanitize_serialized_json(runtime.state) return payload @@ -83,8 +127,8 @@ def _serialize_task(task: TaskModel) -> dict[str, Any]: "id": task.id, "task_id": task.task_id, "name": task.name, - "config": task.config, - "state": task.state, + "config": _sanitize_serialized_json(task.config), + "state": _sanitize_serialized_json(task.state), "ai_thread_id": task.ai_thread_id, "createdAt": task.createdAt, "updatedAt": task.updatedAt, @@ -104,7 +148,7 @@ def _serialize_task_summary(task: TaskModel) -> dict[str, Any]: def _parse_task_config(raw_config: str | None) -> dict[str, Any]: try: - return json.loads(raw_config) + return strip_retired_tracking_fields(json.loads(raw_config)) except Exception as exc: raise TaskServiceError("config格式错误") from exc @@ -116,10 +160,10 @@ async def build_initial_task_state( state_config = await get_state_config(str(PROJECT_ROOT)) base_state = _unwrap_state_config(state_config["config"], task_id) if state_overrides: - base_state = _merge_state(base_state, state_overrides) + base_state = _merge_state(base_state, strip_retired_tracking_fields(state_overrides)) base_state["task_id"] = task_id base_state.setdefault("messages", []) - return base_state + return strip_retired_tracking_fields(base_state) def parse_task_state_overrides(raw_state: str | None) -> dict[str, Any] | None: @@ -128,7 +172,7 @@ def parse_task_state_overrides(raw_state: str | None) -> dict[str, Any] | None: parsed = json.loads(raw_state) if not isinstance(parsed, dict): raise ValueError("state must be a JSON object") - return parsed + return strip_retired_tracking_fields(parsed) async def create_task(task_item: TaskItem) -> dict[str, Any]: @@ -156,10 +200,12 @@ async def get_task(task_id: str) -> dict[str, Any] | None: task = await TaskModel.get_or_none(task_id=task_id) if not task: return None + await _purge_retired_tracking_from_task(task) return _serialize_task(task) async def _load_task_state(task: TaskModel) -> dict[str, Any]: + await _purge_retired_tracking_from_task(task) base_state = await build_initial_task_state(task.task_id) if not task.state: return base_state @@ -169,10 +215,11 @@ async def _load_task_state(task: TaskModel) -> dict[str, Any]: return base_state if not isinstance(current_state, dict): return base_state + current_state = strip_retired_tracking_fields(current_state) merged_state = _merge_state(base_state, current_state) merged_state["task_id"] = task.task_id merged_state.setdefault("messages", []) - return merged_state + return strip_retired_tracking_fields(merged_state) def _parse_state_config_payload(raw_payload: Any) -> dict[str, Any]: @@ -214,6 +261,7 @@ async def update_task_state_config(task_id: str, payload: Any) -> dict[str, Any] states_config = _parse_state_config_payload(payload) state = await _load_task_state(task) apply_state_config_updates(state, states_config) + state = strip_retired_tracking_fields(state) state["task_id"] = task_id state.setdefault("messages", []) @@ -261,14 +309,16 @@ async def delete_task(task_id: str) -> bool: return True -def get_train_status(output_dir: str, task_id: str, train_task_id: str) -> list[Any]: - watch_path = os.path.join(output_dir, task_id, "trainer", train_task_id) - final_path = os.path.join(watch_path, "metrics", "metrics.json") - if not os.path.exists(final_path): - raise TaskServiceError(f"训练状态文件不存在:{final_path}", code=404) +def get_train_status(output_dir: str, task_id: str, train_task_id: str) -> dict[str, Any]: + from loopai.skills.Trainer.results import load_live_training_metrics - with open(final_path, "r") as file_obj: - return json.load(file_obj) + watch_path = Path(output_dir) / task_id / "trainer" / train_task_id + try: + return load_live_training_metrics(watch_path) + except FileNotFoundError as exc: + raise TaskServiceError(str(exc), code=404) from exc + except ValueError as exc: + raise TaskServiceError(str(exc), code=500) from exc async def create_task_runtime( @@ -337,6 +387,7 @@ async def get_latest_task_runtime( ).order_by("-updatedAt", "-id").first() if not runtime: return None + await _purge_retired_tracking_from_runtime(runtime) return _serialize_task_runtime(runtime, include_state=True) @@ -348,6 +399,8 @@ async def list_task_runtime_history( task_id=task_id, node_name=node_name, ).order_by("-updatedAt", "-id") + for runtime in runtimes: + await _purge_retired_tracking_from_runtime(runtime) return [_serialize_task_runtime(runtime, include_state=True) for runtime in runtimes] @@ -362,6 +415,7 @@ async def list_latest_task_runtimes(task_id: str) -> list[dict[str, Any]]: seen_nodes: set[str] = set() for runtime in runtimes: + await _purge_retired_tracking_from_runtime(runtime) node_name = runtime.node_name or "" if node_name in seen_nodes: continue diff --git a/api/app/utils/config/config.py b/api/app/utils/config/config.py index e6f70a4e..6cc0ff08 100644 --- a/api/app/utils/config/config.py +++ b/api/app/utils/config/config.py @@ -5,15 +5,31 @@ from omegaconf import OmegaConf from loopai.schema.states import get_state_config_schema from loopai.schema.system import get_system_config_schema +from loopai.common.tracking import strip_retired_tracking_fields async def check_config_from_db(base_dir): # 判断sqliter中是否有config记录，如果一条也没有，读取./examples/starter.yaml转化为json然后存到数据库 config = await StarterConfig.filter(Q(name='starter')).first() if not config: cfg = OmegaConf.load(os.path.join(base_dir, "starter.yaml")) - config_obj = OmegaConf.to_container(cfg, resolve=True) + config_obj = strip_retired_tracking_fields( + OmegaConf.to_container(cfg, resolve=True) + ) await StarterConfig.create(name='starter', config=json.dumps(config_obj)) config = await StarterConfig.filter(Q(name='starter')).first() + else: + # One-time cleanup for databases created before the Trainer stopped + # storing external tracker credentials in StarterConfig. + try: + original = json.loads(config.config) + cleaned = strip_retired_tracking_fields(original) + if cleaned != original: + config.config = json.dumps(cleaned, ensure_ascii=False) + await config.save(update_fields=["config"]) + except Exception: + # Preserve the existing validation/error behavior for malformed DB + # rows; callers will surface the parse failure in the normal path. + pass return config def wrap_attr(val): @@ -40,8 +56,8 @@ async def get_system_config(base_dir): """获取配置""" config = await check_config_from_db(base_dir) config_data = json.loads(config.config) - system_config = config_data.get('system', {}) - states_data = config_data.get('default_states', {}) + system_config = strip_retired_tracking_fields(config_data.get('system', {})) + states_data = strip_retired_tracking_fields(config_data.get('default_states', {})) language = states_data.get('language', 'zh') if isinstance(states_data, dict) else 'zh' system_schema = get_system_config_schema(language) result = {} @@ -72,7 +88,7 @@ async def get_state_config(base_dir): """获取Starter状态配置""" config = await check_config_from_db(base_dir) config_data = json.loads(config.config) - states_data = config_data.get('default_states', {}) + states_data = strip_retired_tracking_fields(config_data.get('default_states', {})) language = states_data.get('language', 'zh') nested_states_schema = get_state_config_schema(language) default_schema = nested_states_schema.get('default', {}) diff --git a/api/examples/bird.yaml b/api/examples/bird.yaml index 974bf673..72cffdd3 100644 --- a/api/examples/bird.yaml +++ b/api/examples/bird.yaml @@ -25,9 +25,7 @@ save_total_limit: 10 plot_loss: true overwrite_output_dir: true save_only_model: false -report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow] -use_swanlab: true -swanlab_mode: local +report_to: none # local Trainer metric files are used instead ### train per_device_train_batch_size: 1 diff --git a/examples/codex_home_example/AGENTS.md b/examples/codex_home_example/AGENTS.md index 01a07431..234e20f4 100644 --- a/examples/codex_home_example/AGENTS.md +++ b/examples/codex_home_example/AGENTS.md @@ -74,12 +74,16 @@ - `judge` 优先使用 Judger Skill 或 `examples/scripts/run_judger_standalone.py` - 用户要求查看 Judger 过程或评测明细时，优先读取 `judger.pkl` 事件流 - `train` 优先使用 Trainer Skill +- SFT 必须使用 `train_stage=sft, train_framework=llamafactory`；GRPO 必须使用 `train_stage=grpo, train_framework=verl`，不要交叉组合 +- Verl GRPO 默认使用 Conda 环境 `verl`，输入必须是包含 `prompt`、`data_source`、`reward_model` 的训练/验证 Parquet；优先使用 `auto` 或经过用户确认的 LoopAI Reward 预设，只有预设无法覆盖任务时才使用自定义 reward Python 文件 - `obtain`、训练前数据获取、SFT 数据集构造、能力定向提升数据规划，优先读取 Obtainer Skill：`skills/obtainer/SKILL.md` - 涉及 DataMixer、数据湖入湖、SFT recipe/export、按 math/code/text2sql/reasoning 域找数据时，先按 `skills/obtainer/SKILL.md` 和其中的 ObtainerCLI 流程执行，不要从 `outputs/` 里的旧 run 或旧 recipe 反推当前流程 - 执行数据搜集/下载/入湖时，starter 外层只能通过 CLI wrapper 启动 `dataset-acquisition-agent`；如果运行环境不是当前 shell 的 Python，先设置 `LOOPAI_PYTHON_EXECUTABLE=/path/to/loopai-env/bin/python`，再用 `${LOOPAI_PYTHON_EXECUTABLE:-python} -m loopai.skills.ObtainerCLI.cli dm ... dataset-acquisition-agent start`，或在 start 命令上显式传 `--python-executable /path/to/loopai-env/bin/python`；然后轮询/续跑。不要使用通用 `spawn_agent` worker，不要在外层自己创建 SearchAgent task JSON、调用 `searchagent`、调用 `download manifest` 或直接入湖 - 用户要求查看 Trainer 过程或训练事件时，优先读取 Trainer 事件输出 - 每一轮训练都必须先调用 Trainer Skill 的 `prepare()`，向用户完整展示生成的 YAML；只有用户明确确认后才能调用 `run_prepared()`，不得对交互式训练直接调用兼容入口 `run()` - Trainer 必须以前台同步方式运行；训练进入 `completed`、`failed` 或 `cancelled` 前，不得结束当前执行 +- Trainer 使用本地 Skill 执行，Trainer MCP 已禁用，不要启动或调用 Trainer MCP +- Trainer 的训练进程、`trainer.pkl` 更新和结果收尾由持久化 Worker 持有；会话意外结束后不得重复提交同一 version，应通过 `run_state.json`/`worker_result.pkl` 重新接入 - 如果长时间命令返回运行中的 cell/session id，必须持续等待同一执行结束，不能把“训练已启动”当成完成 注意： diff --git a/examples/config/starter.yaml b/examples/config/starter.yaml index ae7e04e3..1c4ba95b 100644 --- a/examples/config/starter.yaml +++ b/examples/config/starter.yaml @@ -148,12 +148,29 @@ default_states: obtainer_debug: false trainer: + # 默认使用 LLaMA-Factory SFT。启用 Verl GRPO 时，将下面两项改为 + # train_framework: "verl" / train_stage: "grpo"，并填写 Parquet 数据路径。 train_framework: "llamafactory" - llamafactory_dir: "/home/lpc/repos/LLaMA-Factory/ + train_stage: "sft" + trainer_persistent_worker: true + llamafactory_dir: "/home/lpc/repos/LLaMA-Factory/" llamafactory_env_path: "/home/lpc/miniconda3/envs/lmf/bin/" CUDA_VISIBLE_DEVICES: "0,1" - swanlab_api_key: "" train_input_dataset_path: "/home/lpc/repos/Dataflow-LoopAI/data/alpaca_en_demo.json" # to defined the path of training dataset (json/jsonl format) train_input_task_description: "训练一个能够回答简单问题和进行对话的AI助手模型，主要用于日常对话和基础问答任务" # to defined the task description for training - train_input_config_template_path: "/home/lpc/repos/Dataflow-LoopAI/api/examples/bird.yaml" # to defined the path of llamafactory config template + train_input_config_template_path: "" # 留空时按 framework/stage 自动选择内置 SFT 或 GRPO 模板 train_input_model_name: "/home/lpc/models/Qwen2.5-0.5B-Instruct/" # to defined the base model name for training + + # Verl GRPO（仅在 train_framework=verl、train_stage=grpo 时使用） + verl_dir: "" # 本地 Verl 仓库根目录，例如 /path/to/verl + verl_env_path: "verl" # Conda 环境名，也可填写环境根目录或 bin 目录 + verl_algorithm: "grpo" + verl_rollout_backend: "vllm" # vllm 或 sglang + verl_model_backend: "fsdp" + train_input_eval_dataset_path: "" # 验证集 Parquet 路径 + verl_reward_mode: "auto" # auto、preset 或 custom + verl_reward_preset: "auto" + verl_reward_kwargs: {} + verl_selection_metric: "val-core/*/acc/mean@*" + verl_selection_mode: "max" + verl_max_actor_ckpt_to_keep: 10 diff --git a/examples/scripts/run_trainer.py b/examples/scripts/run_trainer.py index 0847460e..8ac4506d 100644 --- a/examples/scripts/run_trainer.py +++ b/examples/scripts/run_trainer.py @@ -44,10 +44,6 @@ 'train_input_config_template_path': "loopai/skills/Trainer/templates/qwen2_5_coder_bird_full_sft.yaml", 'train_input_model_name': '/jizhicfs/hymiezhao/models/Qwen2.5-1.5B', 'output_dir': './output/training_test', - - # 可选字段（如果不提供将使用默认值） - 'train_input_use_swanlab': True, - 'train_input_swanlab_project': 'test_llamafactory_training', } } @@ -95,14 +91,11 @@ elif stage_name == 'training_execution' and status: task_id = stage_info.get('task_id') final_status = stage_info.get('final_status') - train_output_swanlab_log_path = stage_info.get('train_output_swanlab_log_path') if task_id: console.print(f" 🆔 训练任务ID: {task_id}") if final_status: status_text = final_status.get('status', '未知') console.print(f" 📊 最终状态: {status_text}") - if train_output_swanlab_log_path: - console.print(f" 📜 SwanLab 日志: {train_output_swanlab_log_path}") if stage_info.get('error'): console.print(f" ❌ 错误: {stage_info['error']}") @@ -125,10 +118,6 @@ if training_info.get('log_path'): console.print(f" 📋 训练日志: {training_info['log_path']}") - # 显示 SwanLab 链接（如果有） - if result.get('swanlab_url'): - console.print(f"\n[bold cyan]📊 SwanLab 监控: {result['swanlab_url']}[/bold cyan]") - except Exception as e: console.print(f"[bold red]❌ 训练流程执行失败: {str(e)}[/bold red]") diff --git a/loopai/common/db_tool/runtime.py b/loopai/common/db_tool/runtime.py index 1fdd4c1b..b4b25c05 100644 --- a/loopai/common/db_tool/runtime.py +++ b/loopai/common/db_tool/runtime.py @@ -1,18 +1,46 @@ from __future__ import annotations +import json import os from typing import Any +from loopai.common.tracking import strip_retired_tracking_fields + from .base import _sqlite_connect, require_db_path +def _sanitize_runtime_state(state: str | None) -> str | None: + if not isinstance(state, str) or not state: + return state + try: + parsed = json.loads(state) + except (TypeError, json.JSONDecodeError): + return state + return json.dumps(strip_retired_tracking_fields(parsed), ensure_ascii=False) + + +def _purge_runtime_rows(con, rows: list[tuple[Any, ...]]) -> list[tuple[Any, ...]]: + cleaned_rows: list[tuple[Any, ...]] = [] + updates: list[tuple[str | None, Any]] = [] + for row in rows: + cleaned_state = _sanitize_runtime_state(row[4]) + if cleaned_state != row[4]: + updates.append((cleaned_state, row[0])) + row = (*row[:4], cleaned_state, *row[5:]) + cleaned_rows.append(row) + if updates: + con.executemany("update taskruntime set state=? where id=?", updates) + con.commit() + return cleaned_rows + + def _serialize_task_runtime_row(row: tuple[Any, ...]) -> dict[str, Any]: return { "id": row[0], "task_id": row[1], "node_name": row[2], "version": row[3], - "state": row[4], + "state": _sanitize_runtime_state(row[4]), "status": row[5], "createdAt": row[6], "updatedAt": row[7], @@ -27,6 +55,7 @@ def create_task_runtime_sync( status: str, state: str | None = None, ) -> dict[str, Any]: + state = _sanitize_runtime_state(state) con = _sqlite_connect(db_path) try: cur = con.execute( @@ -60,6 +89,7 @@ def update_task_runtime_sync( status: str, state: str | None = None, ) -> dict[str, Any] | None: + state = _sanitize_runtime_state(state) con = _sqlite_connect(db_path) try: existing = con.execute( @@ -92,6 +122,8 @@ def update_task_runtime_sync( """, (existing[0],), ).fetchone() + if row is not None: + row = _purge_runtime_rows(con, [row])[0] finally: con.close() if row is None: @@ -167,6 +199,8 @@ def get_latest_task_runtime_sync( """, (task_id, node_name), ).fetchone() + if row is not None: + row = _purge_runtime_rows(con, [row])[0] finally: con.close() if row is None: @@ -191,6 +225,8 @@ def get_current_task_runtime_sync( """, (task_id, version), ).fetchone() + if row is not None: + row = _purge_runtime_rows(con, [row])[0] finally: con.close() if row is None: @@ -214,6 +250,7 @@ def list_task_runtime_history_sync( """, (task_id, node_name), ).fetchall() + rows = _purge_runtime_rows(con, rows) finally: con.close() return [_serialize_task_runtime_row(row) for row in rows] @@ -234,6 +271,7 @@ def list_latest_task_runtimes_sync( """, (task_id,), ).fetchall() + rows = _purge_runtime_rows(con, rows) finally: con.close() diff --git a/loopai/common/db_tool/task.py b/loopai/common/db_tool/task.py index 8140a141..88db0f41 100644 --- a/loopai/common/db_tool/task.py +++ b/loopai/common/db_tool/task.py @@ -7,6 +7,7 @@ from omegaconf import OmegaConf from api.app.models.db_models import StarterConfig +from loopai.common.tracking import strip_retired_tracking_fields from loopai.schema.model_pool import StarterModelPool # 懒加载，避免 MCP 启动时级联导入 langgraph → torch @@ -62,6 +63,16 @@ def format_value(item: dict[str, Any]) -> dict[str, Any]: return item +def _clean_retired_tracking_json(raw_value: str | None) -> str | None: + if not isinstance(raw_value, str) or not raw_value: + return raw_value + original = json.loads(raw_value) + cleaned = strip_retired_tracking_fields(original) + if cleaned == original: + return raw_value + return json.dumps(cleaned, ensure_ascii=False) + + def _normalize_system_config(system_config: dict[str, Any]) -> dict[str, Any]: return _build_system_config(system_config) @@ -69,6 +80,7 @@ def _normalize_system_config(system_config: dict[str, Any]) -> dict[str, Any]: def _build_system_config(system_data: dict[str, Any], language: str | None = None) -> dict[str, Any]: from loopai.schema.system import get_system_config_schema + system_data = strip_retired_tracking_fields(system_data) language = language or "zh" system_schema = get_system_config_schema(language) @@ -96,6 +108,7 @@ def _resolve_model_pool_config(system_config: dict[str, Any], model_name: str | def _build_state_config(states_data: dict[str, Any], language: str | None = None) -> dict[str, Any]: from loopai.schema.states import get_state_config_schema # 懒加载，避免 MCP 级联导入 + states_data = strip_retired_tracking_fields(states_data) language = language or states_data.get("language", "zh") nested_states_schema = get_state_config_schema(language) default_schema = nested_states_schema.get("default", {}) @@ -171,7 +184,9 @@ def _ensure_default_starter_config_sync( return row cfg = OmegaConf.load(str(starter_yaml_path)) - config_obj = OmegaConf.to_container(cfg, resolve=True) + config_obj = strip_retired_tracking_fields( + OmegaConf.to_container(cfg, resolve=True) + ) config_json = json.dumps(config_obj, ensure_ascii=False) cur = con.execute( "insert into starterconfig(name, config) values(?, ?)", @@ -188,15 +203,25 @@ def _get_default_starter_row_sync( starter_yaml_path: str | os.PathLike[str] | None = None, ) -> tuple[int, str, str]: if starter_yaml_path is not None: - return _ensure_default_starter_config_sync(db_path, starter_yaml_path) + row = _ensure_default_starter_config_sync(db_path, starter_yaml_path) + else: + con = _sqlite_connect(db_path) + try: + row = con.execute("select id, name, config from starterconfig where name=?", ("starter",)).fetchone() + finally: + con.close() - con = _sqlite_connect(db_path) - try: - row = con.execute("select id, name, config from starterconfig where name=?", ("starter",)).fetchone() - finally: - con.close() if row is None: raise ValueError("starter config not found") + cleaned_config = _clean_retired_tracking_json(row[2]) + if cleaned_config != row[2]: + con = _sqlite_connect(db_path) + try: + con.execute("update starterconfig set config=? where id=?", (cleaned_config, row[0])) + con.commit() + finally: + con.close() + row = (row[0], row[1], cleaned_config) return row @@ -207,6 +232,16 @@ def _get_task_row_sync(db_path: str | os.PathLike[str], task_id: str) -> tuple[i "select id, task_id, name, config, state from taskmodel where task_id=?", (task_id,), ).fetchone() + if row is not None: + cleaned_config = _clean_retired_tracking_json(row[3]) + cleaned_state = _clean_retired_tracking_json(row[4]) + if cleaned_config != row[3] or cleaned_state != row[4]: + con.execute( + "update taskmodel set config=?, state=? where id=?", + (cleaned_config, cleaned_state, row[0]), + ) + con.commit() + row = (row[0], row[1], row[2], cleaned_config, cleaned_state) finally: con.close() if row is None: @@ -219,7 +254,7 @@ def get_default_system_config_sync( starter_yaml_path: str | os.PathLike[str] | None = None, ) -> dict[str, Any]: config_id, name, raw_config = _get_default_starter_row_sync(db_path, starter_yaml_path) - config_data = json.loads(raw_config or "{}") + config_data = strip_retired_tracking_fields(json.loads(raw_config or "{}")) states_data = config_data.get("default_states", {}) language = states_data.get("language", "zh") if isinstance(states_data, dict) else "zh" return { @@ -235,7 +270,7 @@ def get_default_states_config_sync( section_name: str | None = None, ) -> dict[str, Any]: config_id, name, raw_config = _get_default_starter_row_sync(db_path, starter_yaml_path) - config_data = json.loads(raw_config or "{}") + config_data = strip_retired_tracking_fields(json.loads(raw_config or "{}")) states_config = _build_state_config(config_data.get("default_states", {})) return { "id": config_id, @@ -251,7 +286,7 @@ def update_default_state_section_config_sync( starter_yaml_path: str | os.PathLike[str] | None = None, ) -> dict[str, Any]: config_id, _, raw_config = _get_default_starter_row_sync(db_path, starter_yaml_path) - config_data = json.loads(raw_config or "{}") + config_data = strip_retired_tracking_fields(json.loads(raw_config or "{}")) default_states = config_data.setdefault("default_states", {}) _merge_state_section(default_states, section_name, updates) @@ -270,7 +305,7 @@ def update_default_state_section_config_sync( def get_task_config_sync(db_path: str | os.PathLike[str], task_id: str) -> dict[str, Any]: row_id, row_task_id, name, raw_config, _ = _get_task_row_sync(db_path, task_id) - config_data = json.loads(raw_config or "{}") + config_data = strip_retired_tracking_fields(json.loads(raw_config or "{}")) return { "id": row_id, "task_id": row_task_id, @@ -281,8 +316,8 @@ def get_task_config_sync(db_path: str | os.PathLike[str], task_id: str) -> dict[ def get_task_system_config_sync(db_path: str | os.PathLike[str], task_id: str) -> dict[str, Any]: row_id, row_task_id, name, raw_config, raw_state = _get_task_row_sync(db_path, task_id) - config_data = json.loads(raw_config or "{}") - state_data = json.loads(raw_state or "{}") if raw_state else {} + config_data = strip_retired_tracking_fields(json.loads(raw_config or "{}")) + state_data = strip_retired_tracking_fields(json.loads(raw_state or "{}")) if raw_state else {} if not isinstance(state_data, dict): state_data = {} language = state_data.get("language", "zh") @@ -303,7 +338,7 @@ def get_task_states_config_sync( section_name: str | None = None, ) -> dict[str, Any]: row_id, row_task_id, name, _, raw_state = _get_task_row_sync(db_path, task_id) - state_data = json.loads(raw_state or "{}") if raw_state else {} + state_data = strip_retired_tracking_fields(json.loads(raw_state or "{}")) if raw_state else {} if not isinstance(state_data, dict): state_data = {} states_config = _build_state_config(state_data) @@ -330,7 +365,7 @@ def update_task_state_section_config_sync( updates: dict[str, Any], ) -> dict[str, Any]: row_id, _, _, _, raw_state = _get_task_row_sync(db_path, task_id) - state_data = json.loads(raw_state or "{}") if raw_state else {} + state_data = strip_retired_tracking_fields(json.loads(raw_state or "{}")) if raw_state else {} if not isinstance(state_data, dict): state_data = {} _merge_state_section(state_data, section_name, updates) @@ -401,7 +436,7 @@ def get_default_model_config_sync( starter_yaml_path: str | os.PathLike[str] | None = None, ) -> dict[str, Any]: config_id, name, raw_config = _get_default_starter_row_sync(db_path, starter_yaml_path) - config_data = json.loads(raw_config or "{}") + config_data = strip_retired_tracking_fields(json.loads(raw_config or "{}")) system_config = config_data.get("system", {}) if isinstance(config_data.get("system"), dict) else {} return { "id": config_id, diff --git a/loopai/common/i18n/i18n_dict.py b/loopai/common/i18n/i18n_dict.py index 0b04b8c4..2f793b4c 100644 --- a/loopai/common/i18n/i18n_dict.py +++ b/loopai/common/i18n/i18n_dict.py @@ -1202,10 +1202,6 @@ "en": "CUDA Visible Devices", "zh": "CUDA 可见设备" }, - "SwanLab API Key": { - "en": "SwanLab API Key", - "zh": "SwanLab API Key" - }, "训练数据集路径": { "en": "Training Dataset Path", "zh": "训练数据集路径" @@ -1222,4 +1218,4 @@ "en": "Training Configuration Output Path", "zh": "训练配置输出路径" } -} \ No newline at end of file +} diff --git a/loopai/common/tracking.py b/loopai/common/tracking.py new file mode 100644 index 00000000..15e4e76a --- /dev/null +++ b/loopai/common/tracking.py @@ -0,0 +1,125 @@ +"""Helpers for removing the retired SwanLab integration from persisted inputs.""" + +from __future__ import annotations + +from typing import Any, Mapping + + +_RETIRED_TRACKER_MARKER = "swanlab" +_DROP_VALUE = object() + + +def is_retired_tracking_key(key: Any) -> bool: + """Return whether a mapping key belongs to the retired tracker.""" + return _RETIRED_TRACKER_MARKER in str(key).lower() + + +def is_retired_tracking_value(value: Any) -> bool: + """Return whether a scalar config value selects the retired tracker.""" + return isinstance(value, str) and value.strip().lower() == _RETIRED_TRACKER_MARKER + + +def contains_retired_tracking_reference(value: Any) -> bool: + """Return whether free-form text references the retired tracker.""" + return _RETIRED_TRACKER_MARKER in str(value).lower() + + +def strip_retired_tracking_fields(value: Any) -> Any: + """Return a copy with retired tracker fields removed recursively. + + Old StarterConfig rows, task states, and worker payloads may still contain + fields that are no longer part of the Trainer schema. Filtering by key at + every trust boundary prevents those legacy secrets from being re-exposed. + """ + if isinstance(value, Mapping): + return { + key: strip_retired_tracking_fields(item) + for key, item in value.items() + if not is_retired_tracking_key(key) + } + if isinstance(value, list): + return [strip_retired_tracking_fields(item) for item in value] + if isinstance(value, tuple): + return tuple(strip_retired_tracking_fields(item) for item in value) + return value + + +def _strip_retired_training_config_value(value: Any) -> Any: + if isinstance(value, Mapping): + cleaned: dict[Any, Any] = {} + for key, item in value.items(): + if is_retired_tracking_key(key): + continue + cleaned_item = _strip_retired_training_config_value(item) + if cleaned_item is not _DROP_VALUE: + cleaned[key] = cleaned_item + return cleaned + if isinstance(value, list): + return [ + cleaned_item + for item in value + if (cleaned_item := _strip_retired_training_config_value(item)) is not _DROP_VALUE + ] + if isinstance(value, tuple): + return tuple( + cleaned_item + for item in value + if (cleaned_item := _strip_retired_training_config_value(item)) is not _DROP_VALUE + ) + return _DROP_VALUE if is_retired_tracking_value(value) else value + + +def strip_retired_tracking_config(config: Mapping[str, Any]) -> dict[str, Any]: + """Remove retired tracker keys and logger selections from a config.""" + cleaned = _strip_retired_training_config_value(config) + if not isinstance(cleaned, dict): + raise TypeError("training config must be a mapping") + return cleaned + + +def force_local_training_metrics(config: Mapping[str, Any]) -> dict[str, Any]: + """Return a training config that only uses LoopAI's local metric files.""" + cleaned = strip_retired_tracking_config(config) + cleaned["report_to"] = "none" + return cleaned + + +def strip_retired_tracking_environment(environment: Mapping[str, Any]) -> dict[str, str]: + """Copy an environment without credentials or flags for the retired tracker.""" + return { + str(key): str(value) + for key, value in environment.items() + if not is_retired_tracking_key(key) and str(key).upper() != "TRAIN_USE_SWANLAB" + } + + +def assert_no_retired_tracking(config: Mapping[str, Any]) -> None: + """Reject an old approved YAML that would initialize the retired tracker.""" + retired_keys: list[str] = [] + + def visit(value: Any, prefix: str = "") -> None: + if isinstance(value, Mapping): + for key, item in value.items(): + path = f"{prefix}.{key}" if prefix else str(key) + if is_retired_tracking_key(key): + retired_keys.append(path) + visit(item, path) + elif isinstance(value, (list, tuple)): + for index, item in enumerate(value): + visit(item, f"{prefix}[{index}]") + elif is_retired_tracking_value(value): + retired_keys.append(prefix or "") + + visit(config) + + report_to = config.get("report_to") + report_targets = report_to if isinstance(report_to, (list, tuple)) else [report_to] + if any(_RETIRED_TRACKER_MARKER in str(target).lower() for target in report_targets): + retired_keys.append("report_to") + + if retired_keys: + fields = ", ".join(sorted(set(retired_keys))) + raise ValueError( + "prepared Trainer YAML contains retired SwanLab settings " + f"({fields}); prepare the config again before training" + ) diff --git a/loopai/schema/states.py b/loopai/schema/states.py index b428805b..62e92266 100644 --- a/loopai/schema/states.py +++ b/loopai/schema/states.py @@ -1405,11 +1405,16 @@ class TrainerState(BaseModel): train_framework: str = Field( default="", title="训练框架", - description="训练框架", + description="SFT 使用 llamafactory，GRPO 使用 verl", json_schema_extra={"ui_type": "list", "ui_group": "训练模型", - "allowed_values": ["llamafactory"]}, - # json_schema_extra={"ui_type": "list", "ui_group": "训练模型", - # "allowed_values": ["llamafactory", "verl"]} + "allowed_values": ["llamafactory", "verl"]}, + ) + train_stage: str = Field( + default="", + title="训练阶段", + description="监督微调使用 sft，强化学习使用 grpo", + json_schema_extra={"ui_type": "list", "ui_group": "训练模型", + "allowed_values": ["sft", "grpo"]}, ) llamafactory_dir: str = Field( default="", @@ -1423,18 +1428,111 @@ class TrainerState(BaseModel): description="LlamaFactory 环境路径", json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} ) + verl_dir: str = Field( + default="", + title="Verl 目录", + description="GRPO 使用的 Verl 仓库根目录", + json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} + ) + verl_env_path: str = Field( + default="verl", + title="Verl Conda 环境/路径", + description="默认使用 Conda 环境名 verl，也可填写环境根目录或 bin 目录", + json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} + ) + verl_algorithm: str = Field( + default="grpo", + title="Verl 强化学习算法", + description="当前正式适配仅支持 GRPO", + json_schema_extra={"ui_type": "list", "ui_group": "训练模型", + "allowed_values": ["grpo"]}, + ) + verl_entrypoint: str = Field( + default="verl.trainer.main_ppo", + title="Verl 训练入口", + description="可随固定的 Verl 版本切换 main_ppo/main_ppo_sync", + json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} + ) + verl_rollout_backend: str = Field( + default="vllm", + title="Verl Rollout 后端", + description="GRPO rollout 推理后端", + json_schema_extra={"ui_type": "list", "ui_group": "训练模型", + "allowed_values": ["vllm", "sglang"]}, + ) + verl_model_backend: str = Field( + default="fsdp", + title="Verl 模型训练后端", + description="GRPO actor 的分布式训练后端", + json_schema_extra={"ui_type": "list", "ui_group": "训练模型", + "allowed_values": ["fsdp"]}, + ) + train_input_eval_dataset_path: str = Field( + default="", + title="验证数据集路径", + description="Verl GRPO 使用的验证 Parquet 路径", + json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} + ) + verl_reward_function_path: str = Field( + default="", + title="Verl Reward 函数路径", + description="自定义 GRPO reward Python 文件路径", + json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} + ) + verl_reward_function_name: str = Field( + default="compute_score", + title="Verl Reward 函数名", + description="Reward Python 文件中的可调用函数名", + json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} + ) + verl_reward_mode: str = Field( + default="auto", + title="Verl Reward 模式", + description="auto 自动按 data_source 路由，preset 使用 LoopAI 预设，custom 使用自定义文件", + json_schema_extra={"ui_type": "list", "ui_group": "训练模型", + "allowed_values": ["auto", "preset", "custom"]}, + ) + verl_reward_preset: str = Field( + default="auto", + title="Verl Reward 预设", + description="LoopAI 内置的 Verl Reward 预设", + json_schema_extra={"ui_type": "list", "ui_group": "训练模型", + "allowed_values": [ + "auto", "verl_builtin", "gsm8k_exact", "math_boxed", + "math_dapo", "prime_math", "geometry", "qa_exact_match", + ]}, + ) + verl_reward_kwargs: Dict[str, Any] = Field( + default_factory=dict, + title="Verl Reward 参数", + description="传给 Reward 预设或自定义函数的可选参数", + json_schema_extra={"ui_type": "textarea", "language": "json", "ui_group": "训练模型"}, + ) + verl_selection_metric: str = Field( + default="val-core/*/acc/mean@*", + title="GRPO 最佳模型指标", + description="用于选择最佳 global_step checkpoint 的验证指标模式", + json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} + ) + verl_selection_mode: str = Field( + default="max", + title="GRPO 指标选择方向", + description="Reward/accuracy 通常选择最大值", + json_schema_extra={"ui_type": "list", "ui_group": "训练模型", + "allowed_values": ["max", "min"]}, + ) + verl_max_actor_ckpt_to_keep: int = Field( + default=10, + title="Verl Actor Checkpoint 保留数量", + description="最多保留的 actor checkpoint 数量", + json_schema_extra={"ui_type": "number", "ui_group": "训练模型"} + ) CUDA_VISIBLE_DEVICES: str = Field( default="", title="CUDA 可见设备", description="CUDA 可见设备", json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} ) - swanlab_api_key: str = Field( - default="", - title="SwanLab API Key", - description="SwanLab API Key", - json_schema_extra={"ui_type": "password", "ui_group": "训练模型"} - ) train_input_dataset_path: str = Field( default="", title="训练数据集路径", @@ -1465,18 +1563,12 @@ class TrainerState(BaseModel): description="训练模型名称", json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} ) - train_input_use_swanlab: bool = Field( + trainer_persistent_worker: bool = Field( default=True, - title="是否使用 SwanLab", - description="是否使用 SwanLab", + title="持久化 Trainer Worker", + description="调用方退出后仍继续训练、状态持久化和结果收尾", json_schema_extra={"ui_type": "toggle_switch", "ui_group": "训练模型"} ) - train_input_swanlab_project: str = Field( - default="", - title="SwanLab 项目名称", - description="SwanLab 项目名称", - json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} - ) output_dir: str = Field( default="", title="输出目录", @@ -1581,24 +1673,36 @@ class TrainerState(BaseModel): description="更新模型路径", json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} ) - swanlab_url: str = Field( + trainer_event_log_path: str = Field( default="", - title="SwanLab URL", - description="SwanLab URL", - json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} + title="Trainer 事件日志路径", + description="Trainer 实时事件流持久化文件路径", + json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} ) - train_output_swanlab_log_path: str = Field( + trainer_run_state_path: str = Field( default="", - title="SwanLab 日志路径", - description="SwanLab 日志路径", + title="Trainer Worker 状态路径", + description="持久化 Worker 的 run_state.json 路径", json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} ) - trainer_event_log_path: str = Field( + trainer_worker_log_path: str = Field( default="", - title="Trainer 事件日志路径", - description="Trainer 实时事件流持久化文件路径", + title="Trainer Worker 日志路径", + description="独立 Trainer Worker 的日志路径", json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} ) + trainer_worker_pid: int = Field( + default=0, + title="Trainer Worker PID", + description="持有训练监控和收尾流程的独立进程 PID", + json_schema_extra={"ui_type": "number", "ui_group": "训练模型"} + ) + trainer_state_update_error: str = Field( + default="", + title="Trainer 状态回写错误", + description="训练完成但可选数据库状态回写失败时的诊断信息", + json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} + ) trainer_version_id: str = Field( default="", title="Trainer 运行版本 ID", @@ -1646,6 +1750,12 @@ class TrainerState(BaseModel): json_schema_extra={"ui_type": "textarea", "language": "json", "ui_group": "训练模型"} ) + trainer_result_analysis_version_id: str = Field( + default="", + title="Trainer 结果分析版本", + description="防止同一训练版本重复分析或重复导出 checkpoint", + json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} + ) trainer_result_summary: Dict[str, Any] = Field( default_factory=dict, title="Trainer 结果摘要", @@ -1673,6 +1783,18 @@ class TrainerState(BaseModel): description="根据训练指标选择出的最优 checkpoint 路径", json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} ) + trainer_model_export_error: str = Field( + default="", + title="Verl 模型导出错误", + description="选中 Verl checkpoint 后转换 Hugging Face 模型失败时的错误", + json_schema_extra={"ui_type": "text", "ui_group": "训练模型"} + ) + trainer_model_export_log_path: str = Field( + default="", + title="Verl 模型导出日志", + description="Verl FSDP checkpoint 转换日志路径", + json_schema_extra={"ui_type": "file_path", "ui_group": "训练模型"} + ) train_config: str = Field( default="", title="训练配置", @@ -1940,8 +2062,6 @@ class LoopAIState(MessagesState): # train_input_config_template_path: str # train_config_output_path: str # train_input_model_name: str - # train_input_use_swanlab: bool = True - # train_input_swanlab_project: str # data_check_passed: bool = False # data_check_result: dict = {} # data_check_report_path: str = "" @@ -1959,8 +2079,6 @@ class LoopAIState(MessagesState): # training_service_url: 已废弃，训练现在本地执行 # current_training_status: str = "" # update_model_path: str - # swanlab_url: str - # train_output_swanlab_log_path: str # === Graph Control (图控制属性) === current: str diff --git a/loopai/skills/Trainer/README.md b/loopai/skills/Trainer/README.md index 18450162..5dfb6033 100644 --- a/loopai/skills/Trainer/README.md +++ b/loopai/skills/Trainer/README.md @@ -1,17 +1,92 @@ # Trainer Skill 使用指南 -Trainer Skill 是 Dataflow-LoopAI 中负责模型训练的技能实现。它能够自动化完成从数据验证到模型训练的完整流程，并集成 SwanLab 监控功能。 +Trainer Skill 是 Dataflow-LoopAI 中负责模型训练的技能实现，支持两条互不混用的路径： + +- `sft + llamafactory`：保留原有监督微调流程。 +- `grpo + verl`：使用 Verl 执行 GRPO 强化学习，初版正式支持 FSDP 与 vLLM/SGLang rollout。 + +Trainer 不依赖 MCP。对调用方仍保持同步返回语义，但训练、进度持久化和结果收尾由独立本地 Worker 持有；Codex/API 会话提前结束不会中断 Worker。两条路径都先生成完整 YAML，用户确认后才启动训练。 + +## Verl GRPO 最小配置 + +```python +from loopai.skills.Trainer import prepare, run_prepared + +state = { + "task_id": "my-grpo-task", + "output_dir": "./outputs", + "trainer": { + "train_framework": "verl", + "train_stage": "grpo", + "verl_dir": "/path/to/verl", + "verl_env_path": "verl", # Conda 环境名；默认就是 verl + "train_input_dataset_path": "/data/train.parquet", + "train_input_eval_dataset_path": "/data/validation.parquet", + "train_input_model_name": "/models/base-model", + "train_input_task_description": "Use GRPO to optimize Text2SQL accuracy.", + "verl_reward_mode": "custom", + "verl_reward_function_path": "/data/text2sql_reward.py", + "verl_reward_function_name": "compute_score", + "CUDA_VISIBLE_DEVICES": "0,1,2,3", + }, +} + +prepared = prepare(state=state, thread_id=state["task_id"]) +approval = prepared["trainer"]["trainer_result"]["data"] +# 先向用户展示 approval["config_yaml"]；用户明确确认后： +result = run_prepared( + approval["config_path"], + approval["config_sha256"], + state=prepared, + thread_id=state["task_id"], + version_id=approval["trainer_version_id"], +) +``` + +GRPO 训练/验证 Parquet 至少需要 `prompt`、`data_source`、`reward_model` 三列。训练通过 `conda run --no-capture-output -n verl python -m verl.trainer.main_ppo ...` 启动；指标写入 Trainer 的 `metrics` 目录，checkpoint 按 `global_step_N` 识别，根据 YAML 中的验证指标选择最佳项，再把选中的 FSDP actor 合并为 Hugging Face 模型。 + +## Verl Reward 预设 + +LoopAI 通过 `loopai/skills/Trainer/rewards/router.py` 提供稳定入口，预设只调用当前 Verl 环境中的实现，不复制 Verl 源码。 + +```yaml +trainer: + verl_reward_mode: preset # auto | preset | custom + verl_reward_preset: math_boxed + verl_reward_kwargs: {} +``` + +可用预设： + +- `auto` / `verl_builtin`：根据 Parquet 的 `data_source` 使用 Verl 默认路由；LoopAI 环境安装了 `pyarrow` 时会在准备阶段拒绝未知来源，否则会明确告警并交给 Verl 启动时校验。 +- `gsm8k_exact`：要求输出 `#### answer`，与 `ground_truth` 精确比较。 +- `math_boxed`：提取最后一个 `\\boxed{}` 并进行 MATH 风格等价比较。 +- `math_dapo`：DAPO 数学验证，返回 reward、accuracy 和预测答案。 +- `prime_math`：Numina/PRIME 数学验证。 +- `geometry`：Geometry3K 正确性与格式联合打分。 +- `qa_exact_match`：提取 `...` 并执行规范化精确匹配。 + +`custom` 模式保持原有方式，必须配置 `verl_reward_function_path`；`verl_reward_kwargs` 会传给自定义函数。Trainer 在生成 YAML 前检查训练/验证 Parquet、抽样检查 `prompt`、`data_source`、`reward_model.ground_truth`，并校验预设或自定义函数入口。 ## 🏗️ 架构设计 Trainer Skill 内部采用三阶段顺序执行架构： ``` -数据检查 → 配置生成 → 训练执行 - ↓ ↓ ↓ - 结束 结束 结束 +数据检查 → 配置生成 → 独立 Trainer Worker + ├─ 启动 LLaMAFactory / Verl + ├─ 更新 trainer.pkl / run_state.json + └─ 结果分析、最佳 checkpoint 与模型导出 ``` +默认 `trainer_persistent_worker: true`。调用方存活时会同步等待 Worker 并返回原有结果结构；调用方断开后，Worker 继续运行。每次运行在 `outputs/{task_id}/trainer/{version_id}/` 下新增： + +- `run_state.json`：Worker、训练 PID、当前 step、总 step 和最终状态。 +- `worker.log`：独立 Worker 自身日志。 +- `worker_result.pkl`：用于调用方重连和恢复最终 state，仅本机用户可读。 + +Verl 实时进度优先读取 `metrics/verl_metrics.jsonl` 的 `training/global_step`，不再依赖 tqdm 的回车刷新文本。若需调试时临时恢复旧的会话内执行方式，可显式设置 `trainer_persistent_worker: false`。 + ### 1. 数据检查节点 (Data Check Node) **功能：** 验证数据集格式是否符合 LlamaFactory 要求 @@ -70,12 +145,12 @@ Trainer Skill 内部采用三阶段顺序执行架构： ### 3. 训练执行节点 (Training Execution Node) -**功能：** 执行 LlamaFactory 训练并提供 SwanLab 监控 +**功能：** 执行 LlamaFactory 或 Verl 训练，并通过本地指标文件提供进度和结果 **特性：** - 自动环境验证（Python、CUDA、依赖包） -- 实时训练日志监控 -- SwanLab 集成监控 +- 实时解析本地训练日志和指标文件 +- 不依赖外部实验跟踪服务 - 详细的训练报告生成 ## 📝 使用方法 @@ -96,11 +171,7 @@ training_state = { 'train_input_task_description': '训练一个能够回答简单问题和进行对话的AI助手模型，主要用于日常对话和基础问答任务', 'train_input_config_template_path': "loopai/skills/Trainer/templates/qwen2_5_coder_bird_full_sft.yaml", 'train_input_model_name': '/jizhicfs/hymiezhao/models/Qwen2.5-1.5B', - 'output_dir': './output/trainer_test' - - # 可选字段（如果不提供将使用默认值） - 'train_input_use_swanlab': True, - 'train_input_swanlab_project': 'test_llamafactory_training', + 'output_dir': './output/trainer_test', } # 构建并执行图 @@ -120,8 +191,6 @@ result = graph.invoke(training_state, config=config) | `train_input_model_name` | str | ✅ | - | 基础模型名称 | | `train_input_config_template_path` | str | ✅ | - | 配置模板路径 | | `train_output_dir` | str | ✅ | ./output/training | 训练输出目录 | -| `train_input_use_swanlab` | bool | ❌ | True | 是否使用SwanLab | -| `train_input_swanlab_project` | str | ❌ | llamafactory_training | SwanLab项目名 | | `output_dir` | str | ❌ | ./output/trainer | Agent输出目录 | ### 输出字段 @@ -131,5 +200,8 @@ result = graph.invoke(training_state, config=config) `train_output_data_check_report_path` | str | 数据检查报告路径 | | `train_output_config_path` | str | 生成的配置文件路径 | | `train_output_training_log_path` | str | 训练日志文件路径 | -| `train_output_swanlab_log_path` | str | Swanlog日志路径 | |`train_output_training_report_path` | str | 训练报告路径 | + +训练指标保存在运行目录的本地文件中：SFT 使用 `trainer_log.jsonl` 和 +`metrics/metrics.json`，Verl 使用 `metrics/verl_metrics.jsonl`。前端曲线、结果分析和 +最佳 checkpoint 选择都读取这些文件，不需要外部实验跟踪服务。 diff --git a/loopai/skills/Trainer/nodes/config_generation_node.py b/loopai/skills/Trainer/nodes/config_generation_node.py index 128b85b7..9352b97d 100644 --- a/loopai/skills/Trainer/nodes/config_generation_node.py +++ b/loopai/skills/Trainer/nodes/config_generation_node.py @@ -8,6 +8,11 @@ from pathlib import Path from loopai.schema.states import LoopAIState from loopai.skills.Trainer.utils.config_generator import ConfigGenerator, generate_config_explanation +from loopai.skills.Trainer.utils.verl_config_generator import ( + generate_verl_config_explanation, + generate_verl_grpo_config, + save_verl_grpo_config, +) from loopai.logger import get_logger logger = get_logger() @@ -25,8 +30,6 @@ def config_generation_node(state: LoopAIState) -> LoopAIState: - train_input_dataset_path: 训练数据集路径 - train_input_model_name: 基础模型名称（可选，默认 qwen2.5-7b-instruct） - train_input_config_template_path: 配置模板路径（可选） - - train_input_use_swanlab: 是否使用 SwanLab（可选，默认 True） - - train_input_swanlab_project: SwanLab 项目名称（可选） - output_dir: 输出目录 Returns: @@ -58,8 +61,6 @@ def config_generation_node(state: LoopAIState) -> LoopAIState: training_output_dir = os.path.abspath( state.get('trainer', {}).get('output_dir') or './output/training' ) - use_swanlab = state.get('trainer', {}).get('train_input_use_swanlab', True) - swanlab_project = state.get('trainer', {}).get('train_input_swanlab_project', 'llamafactory_training') framework = state.get('trainer', {}).get('train_framework') @@ -75,8 +76,6 @@ def config_generation_node(state: LoopAIState) -> LoopAIState: model_name=model_name, output_dir=training_output_dir, template_path=template_path, - use_swanlab=use_swanlab, - swanlab_project=swanlab_project ) config["output_dir"] = os.path.abspath(str(config.get("output_dir") or training_output_dir)) @@ -141,17 +140,37 @@ def config_generation_node(state: LoopAIState) -> LoopAIState: logger.info(f" LoRA Rank: {config.get('lora_r')}") logger.info(f" LoRA Alpha: {config.get('lora_alpha')}") - if use_swanlab: - logger.info(f" SwanLab 项目: {swanlab_project}") elif framework == 'verl': - # logger.info("配置生成跳过: Verl 框架当前不支持自动配置生成，请手动提供配置文件") - state.setdefault('trainer', {})['trainer_config_generation_success'] = True - # 直接使用模板配置 + stage = state.get('trainer', {}).get('train_stage') + if stage != 'grpo': + raise ValueError(f"Verl 当前只支持 GRPO，收到 train_stage={stage}") + template_path = state.get('trainer', {}).get('train_input_config_template_path') if not template_path or not os.path.exists(template_path): - raise ValueError("Verl 框架需要提供有效的配置模板路径 (train_input_config_template_path)") - state.setdefault('trainer', {})['train_output_config_path'] = template_path - logger.info("✅ Verl 框架配置生成跳过，已使用提供的配置模板") + raise ValueError("Verl GRPO 需要有效的 YAML 模板路径 (train_input_config_template_path)") + + output_dir = os.path.abspath( + state.get('trainer', {}).get('trainer_output_dir') + or state.get('trainer', {}).get('output_dir') + or './output/trainer' + ) + os.makedirs(output_dir, exist_ok=True) + config_output_path = state.get('trainer', {}).get('train_output_config_path') + if not config_output_path or Path(str(config_output_path)).suffix.lower() not in {'.yaml', '.yml'}: + config_output_path = os.path.join(output_dir, 'training_config.yaml') + + config = generate_verl_grpo_config(state, template_path) + config_output_path = save_verl_grpo_config(config, config_output_path) + explanation_path = os.path.join(output_dir, 'config_explanation.txt') + with open(explanation_path, 'w', encoding='utf-8') as f: + f.write(generate_verl_config_explanation(config)) + + trainer_state = state.setdefault('trainer', {}) + trainer_state['train_config'] = config + trainer_state['train_output_config_path'] = config_output_path + trainer_state['trainer_config_explanation_path'] = explanation_path + trainer_state['trainer_config_generation_success'] = True + logger.info(f"✅ Verl GRPO YAML 已生成: {config_output_path}") else: raise ValueError(f"未知的训练框架: {framework}") diff --git a/loopai/skills/Trainer/nodes/data_check_node.py b/loopai/skills/Trainer/nodes/data_check_node.py index c502a134..45ff8b50 100644 --- a/loopai/skills/Trainer/nodes/data_check_node.py +++ b/loopai/skills/Trainer/nodes/data_check_node.py @@ -8,6 +8,10 @@ from loopai.skills.Trainer.utils.stream_events import prepare_trainer_run from loopai.schema.states import LoopAIState from loopai.skills.Trainer.utils.data_checker import check_data_format, generate_format_report +from loopai.skills.Trainer.utils.verl_data_checker import ( + check_verl_grpo_inputs, + generate_verl_data_report, +) from loopai.logger import get_logger logger = get_logger() @@ -37,7 +41,12 @@ def data_check_node(state: LoopAIState) -> LoopAIState: obtainer_output_file = state.get('obtainer', {}).get('mapping_results', {}).get('output_file') if state.get('obtainer', {}).get('mapping_results') else None constructor_output_file = state.get('constructor', {}).get('mapping_results', {}).get('output_file') if state.get('constructor', {}).get('mapping_results') else None - if obtainer_output_file and os.path.exists(obtainer_output_file): + framework = state.get('trainer', {}).get('train_framework') + if framework == "verl": + # Constructor/Obtainer currently produce SFT JSON. An explicitly supplied + # RL Parquet must not be silently replaced by those artifacts. + dataset_path = state.get('trainer', {}).get('train_input_dataset_path') + elif obtainer_output_file and os.path.exists(obtainer_output_file): dataset_path = obtainer_output_file logger.info(f"使用 obtainer 映射结果作为训练数据集: {dataset_path}") elif constructor_output_file and os.path.exists(constructor_output_file): @@ -54,8 +63,6 @@ def data_check_node(state: LoopAIState) -> LoopAIState: logger.info(f"检查数据集: {dataset_path}") - framework = state.get('trainer', {}).get('train_framework') - if framework == "llamafactory": # 执行数据格式检查 check_result = check_data_format(dataset_path) @@ -95,8 +102,37 @@ def data_check_node(state: LoopAIState) -> LoopAIState: for warning in check_result['warnings'][:3]: # 只显示前3个警告 logger.warning(f" - {warning}") elif framework == "verl": - logger.info("数据检查跳过: Verl 框架当前不支持数据格式检查") - state.setdefault('trainer', {})['trainer_data_check_passed'] = True + trainer_state = state.setdefault('trainer', {}) + if trainer_state.get('train_stage') != 'grpo': + raise ValueError("Verl 当前只支持 GRPO") + eval_path = trainer_state.get('train_input_eval_dataset_path') + if not eval_path: + raise ValueError("Verl GRPO 缺少验证 Parquet (train_input_eval_dataset_path)") + + check_result = check_verl_grpo_inputs( + dataset_path, + eval_path, + trainer_state.get('verl_reward_function_path'), + trainer_state.get('verl_reward_function_name') or 'compute_score', + trainer_state.get('verl_reward_mode'), + trainer_state.get('verl_reward_preset'), + trainer_state.get('verl_reward_kwargs'), + trainer_state.get('verl_dir'), + trainer_state.get('verl_env_path'), + ) + output_dir = trainer_state.get('trainer_output_dir', './output/trainer') + os.makedirs(output_dir, exist_ok=True) + report_path = os.path.join(output_dir, 'data_check_report.txt') + with open(report_path, 'w', encoding='utf-8') as f: + f.write(generate_verl_data_report(check_result)) + trainer_state['trainer_data_check_result'] = check_result + trainer_state['train_output_data_check_report_path'] = report_path + trainer_state['trainer_data_check_passed'] = check_result['is_valid'] + if not check_result['is_valid']: + trainer_state['trainer_data_check_error'] = '; '.join(check_result['errors']) + logger.warning(f"❌ Verl GRPO 数据检查失败: {trainer_state['trainer_data_check_error']}") + else: + logger.info("✅ Verl GRPO 数据、验证集与 reward 预检查通过") else: raise ValueError(f"未知的训练框架: {framework}") diff --git a/loopai/skills/Trainer/nodes/training_execution_node.py b/loopai/skills/Trainer/nodes/training_execution_node.py index 227baac9..92a4a4fb 100644 --- a/loopai/skills/Trainer/nodes/training_execution_node.py +++ b/loopai/skills/Trainer/nodes/training_execution_node.py @@ -4,9 +4,9 @@ """ import os +import hashlib import json import time -import shutil from pathlib import Path from langgraph.config import get_stream_writer from loopai.schema.states import LoopAIState @@ -29,6 +29,53 @@ _task_manager_instance: TaskManager = None +def _materialize_training_config( + source_path: str, + target_path: str, + trainer_state: dict, +) -> str: + """Atomically materialize the exact approved config for the launcher.""" + approved_yaml = trainer_state.pop("_trainer_approved_config_yaml", None) + expected_digest = str( + trainer_state.pop("_trainer_approved_config_sha256", None) or "" + ).strip().lower() + if approved_yaml is not None: + config_bytes = str(approved_yaml).encode("utf-8") + else: + config_bytes = Path(source_path).expanduser().resolve().read_bytes() + + actual_digest = hashlib.sha256(config_bytes).hexdigest() + if expected_digest and actual_digest != expected_digest: + raise ValueError( + "approved Trainer config snapshot digest mismatch: " + f"expected {expected_digest}, got {actual_digest}" + ) + + destination = Path(target_path).expanduser().resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f".{destination.name}.{os.getpid()}.tmp") + try: + with temporary.open("wb") as stream: + stream.write(config_bytes) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary, 0o600) + os.replace(temporary, destination) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + persisted_digest = hashlib.sha256(destination.read_bytes()).hexdigest() + if expected_digest and persisted_digest != expected_digest: + raise ValueError( + "materialized Trainer config digest mismatch: " + f"expected {expected_digest}, got {persisted_digest}" + ) + return str(destination) + + def _get_task_manager(state: dict) -> TaskManager: """ 获取或创建 TaskManager 实例 @@ -54,7 +101,6 @@ def _get_task_manager(state: dict) -> TaskManager: "verl_dir": trainer_state.get('verl_dir') or system_config.get('verl_dir', ''), "llamafactory_env_path": trainer_state.get('llamafactory_env_path') or system_config.get('llamafactory_env_path', ''), "CUDA_VISIBLE_DEVICES": trainer_state.get('CUDA_VISIBLE_DEVICES') or system_config.get('CUDA_VISIBLE_DEVICES', '0,1'), - "swanlab_api_key": trainer_state.get('swanlab_api_key') or system_config.get('swanlab_api_key', ''), "verl_env_path": trainer_state.get('verl_env_path') or system_config.get('verl_env_path', ''), } @@ -69,7 +115,7 @@ def _get_task_manager(state: dict) -> TaskManager: return _task_manager_instance -def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: +def _training_execution_inline(state: LoopAIState, writer=None) -> LoopAIState: """ 训练执行节点 @@ -204,14 +250,19 @@ def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: # 拷贝配置文件到任务管理器的 configs 目录 task_id = trainer_task_id - if framework == 'llamafactory': - config_copy_path = os.path.join(task_manager.configs_dir, f"{task_id}.yaml") - elif framework == 'verl': - config_copy_path = os.path.join(task_manager.configs_dir, f"{task_id}.sh") - else: - config_copy_path = os.path.join(task_manager.configs_dir, f"{task_id}.yaml") - config_copy_path = os.path.abspath(config_copy_path) - shutil.copy(config_path, config_copy_path) + source_suffix = Path(config_path).suffix.lower() + if framework == 'verl' and source_suffix not in {'.yaml', '.yml', '.sh'}: + raise ValueError(f"Verl 配置必须是 YAML: {config_path}") + config_suffix = source_suffix or '.yaml' + config_copy_path = os.path.join(task_manager.configs_dir, f"{task_id}{config_suffix}") + config_copy_path = _materialize_training_config( + config_path, + os.path.abspath(config_copy_path), + state.setdefault('trainer', {}), + ) + # All later consumers, including Verl checkpoint export, must use the + # worker-owned approved snapshot rather than the mutable source path. + state.setdefault('trainer', {})['train_output_config_path'] = config_copy_path # 创建并启动训练任务 logger.info("🚀 提交本地训练任务...") @@ -265,7 +316,13 @@ def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: # 读取训练日志，解析当前进度 log_path = task_manager.get_log_path(task_id) training_progress = None - if os.path.exists(log_path): + if framework == 'verl': + metrics_path = os.path.join(output_dir, "metrics", "verl_metrics.jsonl") + training_progress = TrainingLogParser().parse_verl_metrics_progress( + log_path, + metrics_path, + ) + if training_progress is None and os.path.exists(log_path): temp_parser = TrainingLogParser() training_progress = temp_parser.parse_training_progress(log_path) @@ -300,7 +357,14 @@ def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: "status": task_status, "elapsed_time": int(elapsed_time), "estimated_progress": f"{int(progress_val * 100)}%" - } + } + + training_process = status_info.get('process') + training_pid = getattr(training_process, 'pid', None) + if training_pid is not None: + progress_data["training_pid"] = training_pid + if status_info.get('process_group_id') is not None: + progress_data["process_group_id"] = status_info['process_group_id'] # 实时进度报告 emit_trainer_event( @@ -363,17 +427,6 @@ def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: logs, log_total_lines = read_log_file(log_path, max_lines=1000) log_success = bool(logs) - # 直接从本地获取 SwanLab 日志路径（无需 HTTP 请求） - logger.info("📊 获取SwanLab日志路径...") - swanlab_path = task_manager.get_train_output_swanlab_log_path(task_id) - - if swanlab_path: - logger.info(f"SwanLab日志路径: {swanlab_path}") - state.setdefault('trainer', {})['train_output_swanlab_log_path'] = swanlab_path - else: - logger.warning("SwanLab日志路径未找到") - state.setdefault('trainer', {})['train_output_swanlab_log_path'] = None - # 保存训练日志 os.makedirs(output_dir, exist_ok=True) @@ -397,7 +450,6 @@ def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: data={ "log_retrieved": log_success, "log_lines": log_total_lines, - "swanlab_path": state.get('trainer', {}).get('train_output_swanlab_log_path'), }, ) @@ -407,7 +459,6 @@ def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: final_status=final_status_dict, training_time=training_time, task_description=task_description, - train_output_swanlab_log_path=state.get('trainer', {}).get('train_output_swanlab_log_path'), error=final_status_dict.get('error_message') if final_status_dict else None ) @@ -466,7 +517,28 @@ def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: except Exception as e: logger.warning(f"读取 trainer_log.jsonl 失败: {e}") else: - logger.warning(f"trainer_log.jsonl 不存在: {trainer_log_path}") + if framework != 'verl': + logger.warning(f"trainer_log.jsonl 不存在: {trainer_log_path}") + + if framework == 'verl': + from loopai.skills.Trainer.results import analyze_results + + analysis = analyze_results( + training_output_dir=output_dir, + trainer_state=state.get('trainer', {}), + ) + analysis_data = analysis.get('data') or {} + checkpoints = [ + item.get('name') + for item in analysis_data.get('checkpoints') or [] + if item.get('name') + ] + step_losses = analysis_data.get('metric_records') or [] + logger.info( + f"Verl 结果中发现 {len(checkpoints)} 个 global_step checkpoint，" + f"解析 {len(step_losses)} 条指标" + ) + state.setdefault('trainer', {})['training_checkpoints'] = checkpoints state.setdefault('trainer', {})['training_step_losses'] = step_losses # 进度：训练成功完成 @@ -483,7 +555,6 @@ def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: "training_time": training_time, "report_path": report_path, "log_path": state.get('trainer', {}).get('train_output_training_log_path'), - "train_output_swanlab_log_path": state.get('trainer', {}).get('train_output_swanlab_log_path'), "training_checkpoints": checkpoints, "training_step_losses_count": len(step_losses) }, @@ -554,9 +625,59 @@ def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: return state +def _persistent_worker_enabled(state: LoopAIState) -> bool: + if os.getenv("LOOPAI_TRAINER_WORKER_CHILD") == "1": + return False + value = state.get("trainer", {}).get("trainer_persistent_worker", True) + if isinstance(value, str): + return value.strip().lower() not in {"0", "false", "no", "off"} + return bool(value) + + +def training_execution_node(state: LoopAIState, writer=None) -> LoopAIState: + """Run training in a detached worker while preserving the synchronous API.""" + if not _persistent_worker_enabled(state): + return _training_execution_inline(state, writer=writer) + + if writer is None: + writer = get_stream_writer() + + prepare_trainer_run(state) + emit_trainer_event( + writer, + state, + "training_execution", + "running", + progress=0.0, + message="正在启动持久化 Trainer Worker...", + data={ + "framework": state.get("trainer", {}).get("train_framework"), + "trainer_output_dir": state.get("trainer", {}).get("trainer_output_dir"), + }, + ) + + from loopai.skills.Trainer.utils.persistent_worker import ( + launch_or_attach_persistent_worker, + wait_for_persistent_worker, + ) + + handle = launch_or_attach_persistent_worker(state) + logger.info( + "Trainer Worker 已启动或重新接入: " + f"pid={getattr(handle.process, 'pid', None)}, run_dir={handle.run_dir}" + ) + try: + return wait_for_persistent_worker(handle, stream_writer=writer) + except (KeyboardInterrupt, SystemExit): + # The worker owns the training lifecycle. Do not cancel it merely + # because the Codex/API caller has disconnected. + logger.warning(f"Trainer 调用方退出；Worker 将继续运行: {handle.run_dir}") + raise + + def _generate_training_report(task_id: str, final_status: dict, training_time: float, task_description: str, - train_output_swanlab_log_path: str = None, error: str = None) -> str: + error: str = None) -> str: """ 生成训练报告 @@ -565,7 +686,6 @@ def _generate_training_report(task_id: str, final_status: dict, final_status: 最终状态信息 training_time: 训练用时 task_description: 任务描述 - train_output_swanlab_log_path: SwanLab日志路径 error: 错误信息 Returns: @@ -584,11 +704,6 @@ def _generate_training_report(task_id: str, final_status: dict, report.append(f" 任务描述: {task_description}") report.append(f" 执行时间: {training_time:.2f} 秒") - if train_output_swanlab_log_path: - report.append(f" SwanLab日志路径: {train_output_swanlab_log_path}") - else: - report.append(" SwanLab日志路径: 未找到") - report.append("") # 状态信息 @@ -614,10 +729,6 @@ def _generate_training_report(task_id: str, final_status: dict, report.append("✅ 训练执行成功") report.append("- 训练任务已成功完成") report.append("- 模型已保存到指定输出目录") - if train_output_swanlab_log_path: - report.append(f"- SwanLab训练监控日志: {train_output_swanlab_log_path}") - else: - report.append("- SwanLab训练监控日志: 未找到或未生成") elif error: report.append("❌ 训练执行失败") report.append(f"- 错误原因: {error}") @@ -632,9 +743,6 @@ def _generate_training_report(task_id: str, final_status: dict, report.append("注意事项:") report.append("- 训练结果保存在本地输出目录中") report.append("- 训练日志和监控数据可在输出目录中查看") - if train_output_swanlab_log_path: - report.append("- SwanLab实验监控数据可在指定路径查看") - report.append("- 可通过SwanLab界面查看训练曲线和指标") return "\n".join(report) @@ -655,8 +763,6 @@ def get_training_status(state: LoopAIState) -> dict: "trainer_config_generation_success": state.get('trainer', {}).get('trainer_config_generation_success', False), "trainer_training_success": state.get('trainer', {}).get('trainer_training_success', False), "training_time": state.get('trainer', {}).get('trainer_training_execution_time', 0), - "swanlab_url": state.get('trainer', {}).get('swanlab_url', ''), - "train_output_swanlab_log_path": state.get('trainer', {}).get('train_output_swanlab_log_path'), "train_output_training_log_path": state.get('trainer', {}).get('train_output_training_log_path'), "output_dir": state.get('trainer', {}).get('train_output_dir'), "errors": [] diff --git a/loopai/skills/Trainer/results.py b/loopai/skills/Trainer/results.py index fbb26966..1de1bb28 100644 --- a/loopai/skills/Trainer/results.py +++ b/loopai/skills/Trainer/results.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import math +from fnmatch import fnmatch from pathlib import Path from typing import Any, Dict, Iterable, List, Optional @@ -8,38 +10,67 @@ def _to_number(value: Any) -> float | int | None: if isinstance(value, bool) or value is None: return None - if isinstance(value, (int, float)): + if isinstance(value, int): return value - try: - number = float(str(value)) - except (TypeError, ValueError): + if isinstance(value, float): + number = value + else: + try: + number = float(str(value)) + except (TypeError, ValueError): + return None + if not math.isfinite(number): return None return int(number) if number.is_integer() else number +def _normalize_selection_mode(value: Any) -> str: + mode = str(value or "max").strip().lower() + if mode not in {"max", "min"}: + raise ValueError(f"selection_mode must be max or min, got: {value}") + return mode + + def _checkpoint_step(name: str) -> int | None: - if not name.startswith("checkpoint-"): - return None - return int(name.split("-", 1)[1]) if name.split("-", 1)[1].isdigit() else None + for prefix in ("checkpoint-", "global_step_"): + if name.startswith(prefix): + suffix = name[len(prefix):] + return int(suffix) if suffix.isdigit() else None + return None def _normalize_metric_record(record: Dict[str, Any], source: str) -> Dict[str, Any] | None: - step = _to_number(record.get("step", record.get("current_steps", record.get("global_step")))) - loss = _to_number(record.get("loss")) - eval_loss = _to_number(record.get("eval_loss")) - if loss is None and eval_loss is None: - return None - + nested_data = record.get("data") + flattened = dict(nested_data) if isinstance(nested_data, dict) else dict(record) + if isinstance(nested_data, dict) and "step" not in flattened: + flattened["step"] = record.get("step") + + step = _to_number(flattened.get( + "step", + flattened.get("current_steps", flattened.get("global_step", flattened.get("training/global_step"))), + )) normalized: Dict[str, Any] = {"source": source} if step is not None: normalized["step"] = int(step) - if loss is not None: - normalized["loss"] = float(loss) - if eval_loss is not None: - normalized["eval_loss"] = float(eval_loss) - for key in ("epoch", "lr", "learning_rate", "timestamp"): - if key in record: - normalized[key] = record[key] + for key, value in flattened.items(): + if key in {"step", "current_steps", "global_step", "timestamp", "log_line", "data"}: + continue + number = _to_number(value) + if number is not None: + normalized[key] = number + if "training/epoch" in normalized and "epoch" not in normalized: + normalized["epoch"] = normalized["training/epoch"] + if "critic/rewards/mean" in normalized and "reward" not in normalized: + normalized["reward"] = normalized["critic/rewards/mean"] + if "actor/pg_loss" in normalized and "policy_loss" not in normalized: + normalized["policy_loss"] = normalized["actor/pg_loss"] + val_keys = sorted(key for key in normalized if key.startswith("val-core/")) + if val_keys and "val_score" not in normalized: + normalized["val_score"] = normalized[val_keys[0]] + if "timestamp" in record: + normalized["timestamp"] = record["timestamp"] + if len(normalized) == 1 or (len(normalized) == 2 and "step" in normalized): + return None return normalized @@ -77,7 +108,7 @@ def _iter_metric_items(payload: Any) -> Iterable[Dict[str, Any]]: for item in metrics: if isinstance(item, dict): yield item - elif all(key in payload for key in ("step", "loss")) or "eval_loss" in payload: + elif "step" in payload or "eval_loss" in payload or "training/global_step" in payload: yield payload @@ -99,11 +130,33 @@ def _read_metrics_json(path: Path) -> List[Dict[str, Any]]: return records +def _read_verl_jsonl_metrics(path: Path) -> List[Dict[str, Any]]: + if not path.is_file(): + return [] + records: List[Dict[str, Any]] = [] + with path.open("r", encoding="utf-8") as f: + for line in f: + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(item, dict): + normalized = _normalize_metric_record(item, "verl_file_logger") + if normalized is not None: + records.append(normalized) + return records + + def _dedupe_metrics(records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - seen: set[tuple[Any, Any, Any]] = set() + seen: set[str] = set() deduped: List[Dict[str, Any]] = [] for record in records: - key = (record.get("step"), record.get("loss"), record.get("eval_loss")) + key = json.dumps( + {k: v for k, v in record.items() if k not in {"source", "timestamp"}}, + sort_keys=True, + ensure_ascii=False, + default=str, + ) if key in seen: continue seen.add(key) @@ -111,6 +164,62 @@ def _dedupe_metrics(records: List[Dict[str, Any]]) -> List[Dict[str, Any]]: return sorted(deduped, key=lambda item: (item.get("step") is None, item.get("step") or 0)) +def load_live_training_metrics(training_output_dir: str | Path) -> Dict[str, Any]: + """Return the UI metrics contract for either Verl or LLaMAFactory.""" + root = Path(training_output_dir).expanduser().resolve() + metrics_dir = root / "metrics" + verl_metrics_path = metrics_dir / "verl_metrics.jsonl" + metrics_path = metrics_dir / "metrics.json" + + if verl_metrics_path.is_file(): + records = _dedupe_metrics(_read_verl_jsonl_metrics(verl_metrics_path)) + + # Keep the existing terminal-log panel without feeding the generic + # console parser's noisy PID/Progress fields into chart selection. + log_records: List[Dict[str, Any]] = [] + if metrics_path.is_file(): + try: + with metrics_path.open("r", encoding="utf-8") as file: + raw_metrics = json.load(file) + except (OSError, json.JSONDecodeError): + raw_metrics = {} + for item in _iter_metric_items(raw_metrics): + if item.get("log_line"): + log_records.append({"log_line": item["log_line"]}) + + return { + "task_info": { + "framework": "verl", + "total_metrics": len(records), + "metrics_source": str(verl_metrics_path), + }, + "metrics": records + log_records[-200:], + } + + if metrics_path.is_file(): + try: + with metrics_path.open("r", encoding="utf-8") as file: + payload = json.load(file) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"invalid Trainer metrics file: {metrics_path}") from exc + if isinstance(payload, dict): + return payload + if isinstance(payload, list): + return { + "task_info": { + "framework": "llamafactory", + "total_metrics": len(payload), + "metrics_source": str(metrics_path), + }, + "metrics": payload, + } + raise ValueError(f"invalid Trainer metrics payload: {metrics_path}") + + raise FileNotFoundError( + f"Trainer metrics do not exist: {verl_metrics_path} or {metrics_path}" + ) + + def _find_training_output_dir( *, task_id: str | None = None, @@ -154,36 +263,112 @@ def _find_training_output_dir( if candidate.is_dir() and ( (candidate / "trainer_log.jsonl").exists() or (candidate / "metrics" / "metrics.json").exists() - or any(child.name.startswith("checkpoint-") for child in candidate.iterdir()) + or (candidate / "metrics" / "verl_metrics.jsonl").exists() + or any(_checkpoint_step(child.name) is not None for child in candidate.iterdir()) + or ( + (candidate / "checkpoints").is_dir() + and any(_checkpoint_step(child.name) is not None for child in (candidate / "checkpoints").iterdir()) + ) ): return candidate return None -def _load_checkpoints(training_output_dir: Path) -> List[Dict[str, Any]]: +def _checkpoint_roots(training_output_dir: Path, trainer_state: Optional[Dict[str, Any]]) -> List[Path]: + roots: List[Path] = [] + config = (trainer_state or {}).get("train_config") or {} + overrides = config.get("overrides") if isinstance(config, dict) else {} + configured = overrides.get("trainer.default_local_dir") if isinstance(overrides, dict) else None + for candidate in ( + Path(str(configured)) if configured else None, + training_output_dir, + training_output_dir / "checkpoints", + ): + if candidate and candidate.is_dir() and candidate not in roots: + roots.append(candidate) + return roots + + +def _has_huggingface_weights(path: Path) -> bool: + weight_names = { + "model.safetensors", + "model.safetensors.index.json", + "pytorch_model.bin", + "pytorch_model.bin.index.json", + } + return path.is_dir() and any((path / name).is_file() for name in weight_names) + + +def _load_checkpoints( + training_output_dir: Path, + trainer_state: Optional[Dict[str, Any]] = None, +) -> List[Dict[str, Any]]: checkpoints: List[Dict[str, Any]] = [] - children = training_output_dir.iterdir() if training_output_dir.is_dir() else [] - for child in children: - if not child.is_dir() or not child.name.startswith("checkpoint-"): - continue - checkpoints.append({ - "name": child.name, - "step": _checkpoint_step(child.name), - "path": str(child), - }) + seen: set[str] = set() + for root in _checkpoint_roots(training_output_dir, trainer_state): + for child in root.iterdir(): + step = _checkpoint_step(child.name) + if not child.is_dir() or step is None or str(child) in seen: + continue + seen.add(str(child)) + actor_path = child / "actor" if child.name.startswith("global_step_") else child + huggingface_path = actor_path / "huggingface" + has_huggingface_weights = _has_huggingface_weights(huggingface_path) + model_path = huggingface_path if has_huggingface_weights else actor_path + checkpoints.append({ + "name": child.name, + "step": step, + "path": str(model_path), + "checkpoint_path": str(child), + "backend": "verl" if child.name.startswith("global_step_") else "llamafactory", + "needs_merge": child.name.startswith("global_step_") and not has_huggingface_weights, + }) return sorted(checkpoints, key=lambda item: item.get("step") if item.get("step") is not None else -1) -def _best_metric(records: List[Dict[str, Any]]) -> Dict[str, Any] | None: +def _best_metric( + records: List[Dict[str, Any]], + selection_metric: str | None = None, + selection_mode: str | None = None, +) -> Dict[str, Any] | None: + normalized_mode = _normalize_selection_mode(selection_mode) + if selection_metric: + grouped: List[Dict[str, Any]] = [] + for record in records: + matched = {} + for key, value in record.items(): + number = _to_number(value) + if fnmatch(key, selection_metric) and number is not None: + matched[key] = float(number) + if matched: + step = _to_number(record.get("step")) + grouped.append({ + "metric": selection_metric, + "value": sum(matched.values()) / len(matched), + "step": int(step) if step is not None else None, + "source": record.get("source"), + "matched_metrics": matched, + "record": record, + }) + if grouped: + if normalized_mode == "max": + return max(grouped, key=lambda item: (item["value"], item.get("step") or 0)) + return min(grouped, key=lambda item: (item["value"], -(item.get("step") or 0))) + candidates: List[Dict[str, Any]] = [] + metric_order = ("eval_loss", "loss", "val_score", "reward") for record in records: - for metric_name in ("eval_loss", "loss"): + for metric_name in metric_order: if metric_name not in record: continue + value = _to_number(record[metric_name]) + if value is None: + continue + step = _to_number(record.get("step")) candidates.append({ "metric": metric_name, - "value": record[metric_name], - "step": record.get("step"), + "value": value, + "step": int(step) if step is not None else None, "source": record.get("source"), "record": record, }) @@ -191,30 +376,33 @@ def _best_metric(records: List[Dict[str, Any]]) -> Dict[str, Any] | None: if not candidates: return None - return sorted( - candidates, - key=lambda item: ( - 0 if item["metric"] == "eval_loss" else 1, - item["value"], - -(item.get("step") or 0), - ), - )[0] + for metric_name in metric_order: + metric_candidates = [item for item in candidates if item["metric"] == metric_name] + if not metric_candidates: + continue + maximize = metric_name in {"val_score", "reward"} + if maximize: + return max(metric_candidates, key=lambda item: (item["value"], item.get("step") or 0)) + return min(metric_candidates, key=lambda item: (item["value"], -(item.get("step") or 0))) + return None def select_best_checkpoint( checkpoints: List[Dict[str, Any]], metric_records: List[Dict[str, Any]], + selection_metric: str | None = None, + selection_mode: str | None = None, ) -> Dict[str, Any] | None: - """Select the best checkpoint by eval_loss first, then train loss, then latest checkpoint.""" + """Select a checkpoint and align metric steps to persisted checkpoint steps.""" if not checkpoints: return None - metric = _best_metric(metric_records) + metric = _best_metric(metric_records, selection_metric, selection_mode) if metric is None or metric.get("step") is None: checkpoint = checkpoints[-1] return { **checkpoint, - "selection_reason": "No usable loss metric found; selected the latest checkpoint.", + "selection_reason": "No usable selection metric found; selected the latest checkpoint.", "metric": None, } @@ -240,7 +428,7 @@ def select_best_checkpoint( "selection_reason": reason, "metric": { key: metric[key] - for key in ("metric", "value", "step", "source") + for key in ("metric", "value", "step", "source", "matched_metrics") if key in metric }, } @@ -277,10 +465,33 @@ def analyze_results( trainer_log_path = resolved_dir / "trainer_log.jsonl" metrics_path = resolved_dir / "metrics" / "metrics.json" - records = _dedupe_metrics(_read_jsonl_metrics(trainer_log_path) + _read_metrics_json(metrics_path)) - checkpoints = _load_checkpoints(resolved_dir) - best_checkpoint = select_best_checkpoint(checkpoints, records) - best_metric = best_checkpoint.get("metric") if best_checkpoint else _best_metric(records) + verl_metrics_path = resolved_dir / "metrics" / "verl_metrics.jsonl" + records = _dedupe_metrics( + _read_jsonl_metrics(trainer_log_path) + + _read_metrics_json(metrics_path) + + _read_verl_jsonl_metrics(verl_metrics_path) + ) + state = trainer_state or {} + train_config = state.get("train_config") or {} + result_config = train_config.get("result") if isinstance(train_config, dict) else {} + # The approved YAML is the source of truth. Runtime defaults only fill + # fields that the user did not set in the reviewed configuration. + selection_metric = ( + result_config.get("selection_metric") if isinstance(result_config, dict) else None + ) or state.get("verl_selection_metric") + selection_mode = ( + result_config.get("selection_mode") if isinstance(result_config, dict) else None + ) or state.get("verl_selection_mode") + checkpoints = _load_checkpoints(resolved_dir, trainer_state) + best_checkpoint = select_best_checkpoint( + checkpoints, + records, + selection_metric=selection_metric, + selection_mode=selection_mode, + ) + best_metric = best_checkpoint.get("metric") if best_checkpoint else _best_metric( + records, selection_metric, selection_mode + ) eval_records = [item for item in records if "eval_loss" in item] loss_records = [item for item in records if "loss" in item] @@ -291,6 +502,7 @@ def analyze_results( "metric_count": len(records), "eval_loss_count": len(eval_records), "loss_count": len(loss_records), + "reward_count": len([item for item in records if "reward" in item]), "latest_metric": latest_metric, "best_checkpoint_name": best_checkpoint.get("name") if best_checkpoint else None, "best_checkpoint_path": best_checkpoint.get("path") if best_checkpoint else None, @@ -307,6 +519,7 @@ def analyze_results( "training_output_dir": str(resolved_dir), "trainer_log_path": str(trainer_log_path) if trainer_log_path.exists() else None, "metrics_path": str(metrics_path) if metrics_path.exists() else None, + "verl_metrics_path": str(verl_metrics_path) if verl_metrics_path.exists() else None, "checkpoints": checkpoints, "metric_records": records, "best_checkpoint": best_checkpoint, diff --git a/loopai/skills/Trainer/rewards/__init__.py b/loopai/skills/Trainer/rewards/__init__.py new file mode 100644 index 00000000..1f203d48 --- /dev/null +++ b/loopai/skills/Trainer/rewards/__init__.py @@ -0,0 +1,19 @@ +"""LoopAI reward presets for Verl-backed reinforcement learning.""" + +from .router import ( + REWARD_PRESETS, + SUPPORTED_REWARD_PRESETS, + compute_score, + get_reward_preset, + is_verl_builtin_data_source, + normalize_reward_preset, +) + +__all__ = [ + "REWARD_PRESETS", + "SUPPORTED_REWARD_PRESETS", + "compute_score", + "get_reward_preset", + "is_verl_builtin_data_source", + "normalize_reward_preset", +] diff --git a/loopai/skills/Trainer/rewards/router.py b/loopai/skills/Trainer/rewards/router.py new file mode 100644 index 00000000..7b80bedd --- /dev/null +++ b/loopai/skills/Trainer/rewards/router.py @@ -0,0 +1,208 @@ +"""Stable LoopAI entry point for reward implementations provided by Verl. + +Verl loads this file directly from the generated training YAML. Keep imports of +Verl lazy so LoopAI can inspect configs without installing the Verl environment +in the API process. +""" + +from __future__ import annotations + +from typing import Any, Callable, Dict, Iterable + + +REWARD_PRESETS: Dict[str, Dict[str, Any]] = { + "auto": { + "description": "Let Verl select its built-in reward from data_source.", + "ground_truth": "Depends on the selected Verl reward.", + "data_sources": ["Verl-supported data_source values"], + }, + "verl_builtin": { + "description": "Explicit alias for Verl's default data_source router.", + "ground_truth": "Depends on the selected Verl reward.", + "data_sources": ["Verl-supported data_source values"], + }, + "gsm8k_exact": { + "description": "GSM8K final-answer exact match using Verl's implementation.", + "ground_truth": "Final numeric answer as a string.", + "data_sources": ["openai/gsm8k"], + }, + "math_boxed": { + "description": "Last boxed-answer equivalence for MATH-style datasets.", + "ground_truth": "Reference mathematical answer as a string.", + "data_sources": [ + "lighteval/MATH", + "DigitalLearningGmbH/MATH-lighteval", + "HuggingFaceH4/MATH-500", + ], + }, + "math_dapo": { + "description": "DAPO mathematical verification with score and accuracy details.", + "ground_truth": "Reference mathematical answer as a string.", + "data_sources": ["math_dapo", "math", "math_dapo_reasoning", "aime*"], + }, + "prime_math": { + "description": "PRIME/Numina mathematical answer verification from Verl.", + "ground_truth": "Reference mathematical answer as a string.", + "data_sources": ["numina_*"], + }, + "geometry": { + "description": "Geometry3K correctness plus boxed-output format reward.", + "ground_truth": "Reference geometry answer as a string.", + "data_sources": ["hiyouga/geometry3k"], + }, + "qa_exact_match": { + "description": "Search-R1 answer-tag extraction and normalized exact match.", + "ground_truth": "A mapping containing target, whose value is a string or list.", + "data_sources": ["searchR1_*"], + }, +} + +SUPPORTED_REWARD_PRESETS = tuple(REWARD_PRESETS) + +_ALIASES = { + "builtin": "verl_builtin", + "default": "auto", + "gsm8k": "gsm8k_exact", + "math": "math_boxed", + "math_reward": "math_boxed", + "dapo": "math_dapo", + "geo3k": "geometry", + "qa_em": "qa_exact_match", + "search_r1": "qa_exact_match", +} + +_VERL_BUILTIN_EXACT_DATA_SOURCES = { + "openai/gsm8k", + "lighteval/MATH", + "DigitalLearningGmbH/MATH-lighteval", + "HuggingFaceH4/MATH-500", + "math_dapo", + "math", + "math_dapo_reasoning", + "numina_aops_forum", + "numina_synthetic_math", + "numina_amc_aime", + "numina_synthetic_amc", + "numina_cn_k12", + "numina_olympiads", + "codecontests", + "apps", + "codeforces", + "taco", + "hiyouga/geometry3k", + "searchR1_nq", + "searchR1_triviaqa", + "searchR1_popqa", + "searchR1_hotpotqa", + "searchR1_2wikimultihopqa", + "searchR1_musique", + "searchR1_bamboogle", +} + + +def normalize_reward_preset(value: Any) -> str: + """Normalize and validate a user-facing reward preset name.""" + preset = str(value or "auto").strip().lower().replace("-", "_") + preset = _ALIASES.get(preset, preset) + if preset not in REWARD_PRESETS: + supported = ", ".join(SUPPORTED_REWARD_PRESETS) + raise ValueError(f"Unsupported LoopAI reward preset: {preset}; expected one of: {supported}") + return preset + + +def get_reward_preset(value: Any = "auto") -> Dict[str, Any]: + """Return serializable metadata for one normalized preset.""" + preset = normalize_reward_preset(value) + return {"name": preset, **REWARD_PRESETS[preset]} + + +def is_verl_builtin_data_source(data_source: Any) -> bool: + """Whether the pinned Verl default router recognizes this data source.""" + value = str(data_source or "").strip() + return value in _VERL_BUILTIN_EXACT_DATA_SOURCES or value.startswith("aime") + + +def _kwargs(source: Dict[str, Any], allowed: Iterable[str]) -> Dict[str, Any]: + return {key: source[key] for key in allowed if key in source} + + +def _normalize_result(result: Any) -> float | Dict[str, Any]: + """Match the scalar/dict contract expected by Verl reward managers.""" + if isinstance(result, dict): + if "score" not in result: + raise ValueError("Reward result mappings must contain a 'score' field") + return result + if isinstance(result, (list, tuple)): + if not result: + raise ValueError("Reward result sequence must not be empty") + result = result[0] + if isinstance(result, (bool, int, float)): + return float(result) + raise TypeError(f"Unsupported reward result type: {type(result).__name__}") + + +def compute_score( + data_source: str, + solution_str: str, + ground_truth: Any, + extra_info: Dict[str, Any] | None = None, + preset: str = "auto", + **kwargs: Any, +) -> float | Dict[str, Any]: + """Compute one reward using a named LoopAI preset backed by Verl. + + The signature intentionally matches Verl's ``NaiveRewardManager``. Named + presets ignore ``data_source`` for routing, while ``auto`` and + ``verl_builtin`` delegate routing to Verl. + """ + resolved = normalize_reward_preset(preset) + + if resolved in {"auto", "verl_builtin"}: + from verl.utils.reward_score import default_compute_score + + return _normalize_result( + default_compute_score( + data_source=data_source, + solution_str=solution_str, + ground_truth=ground_truth, + extra_info=extra_info, + **kwargs, + ) + ) + + scorer: Callable[..., Any] + scorer_kwargs: Dict[str, Any] + if resolved == "gsm8k_exact": + from verl.utils.reward_score import gsm8k + + scorer = gsm8k.compute_score + scorer_kwargs = _kwargs(kwargs, ("method", "format_score", "score")) + elif resolved == "math_boxed": + from verl.utils.reward_score import math_reward + + scorer = math_reward.compute_score + scorer_kwargs = {} + elif resolved == "math_dapo": + from verl.utils.reward_score import math_dapo + + scorer = math_dapo.compute_score + scorer_kwargs = _kwargs(kwargs, ("strict_box_verify", "pause_tokens_index")) + elif resolved == "prime_math": + from verl.utils.reward_score import prime_math + + scorer = prime_math.compute_score + scorer_kwargs = {} + elif resolved == "geometry": + from verl.utils.reward_score import geo3k + + scorer = geo3k.compute_score + scorer_kwargs = _kwargs(kwargs, ("use_boxed", "format_score")) + elif resolved == "qa_exact_match": + from verl.utils.reward_score import search_r1_like_qa_em + + scorer = search_r1_like_qa_em.compute_score + scorer_kwargs = _kwargs(kwargs, ("method", "format_score", "score")) + else: # pragma: no cover - normalize_reward_preset guards this branch. + raise AssertionError(f"Unhandled LoopAI reward preset: {resolved}") + + return _normalize_result(scorer(solution_str, ground_truth, **scorer_kwargs)) diff --git a/loopai/skills/Trainer/runner.py b/loopai/skills/Trainer/runner.py index 697adca4..22b7929f 100644 --- a/loopai/skills/Trainer/runner.py +++ b/loopai/skills/Trainer/runner.py @@ -1,12 +1,14 @@ from __future__ import annotations import hashlib +import json import os from pathlib import Path from typing import Any, Dict, Optional import yaml +from loopai.common.tracking import assert_no_retired_tracking, strip_retired_tracking_fields from loopai.common.exception import ErrorCode, emit_error, emit_success from loopai.skills.Trainer.runtime_config import ( build_trainer_prefill_guide, @@ -33,9 +35,12 @@ "training_error", "current_training_status", "update_model_path", - "swanlab_url", - "train_output_swanlab_log_path", "trainer_event_log_path", + "trainer_run_state_path", + "trainer_worker_log_path", + "trainer_worker_pid", + "trainer_persistent_worker", + "trainer_state_update_error", "trainer_version_id", "trainer_output_dir", "trainer_missing_fields", @@ -43,10 +48,13 @@ "trainer_result", "trainer_last_error", "trainer_result_analysis", + "trainer_result_analysis_version_id", "trainer_result_summary", "trainer_best_checkpoint", "trainer_best_metric", "trainer_best_checkpoint_path", + "trainer_model_export_error", + "trainer_model_export_log_path", "train_config", "training_checkpoints", "training_step_losses", @@ -176,6 +184,7 @@ def inspect_prepared_trainer_config(config_path: str) -> Dict[str, Any]: parsed = yaml.safe_load(config_yaml) if not isinstance(parsed, dict): raise ValueError(f"prepared Trainer config must contain a YAML mapping: {path}") + assert_no_retired_tracking(parsed) return { "config_path": str(path), @@ -211,6 +220,103 @@ def _update_trainer_task_state(runtime: Dict[str, Any], trainer_state: Dict[str, raise RuntimeError(detail or "failed to update Trainer task state config") +def finalize_trainer_result_state( + result: Dict[str, Any], + *, + task_id: Optional[str] = None, +) -> Dict[str, Any]: + """Analyze artifacts and export a Verl checkpoint in an idempotent way.""" + if not isinstance(result, dict): + return result + + trainer_state = result.setdefault("trainer", {}) + version_id = str(trainer_state.get("trainer_version_id") or "") + if ( + isinstance(trainer_state.get("trainer_result_analysis"), dict) + and str(trainer_state.get("trainer_result_analysis_version_id") or "") == version_id + ): + return result + try: + from loopai.skills.Trainer.results import analyze_results + + analysis_payload = analyze_results( + task_id=task_id or result.get("task_id"), + output_dir=result.get("output_dir", "./outputs"), + trainer_state=trainer_state, + ) + analysis_data = analysis_payload.get("data") or {} + summary = analysis_data.get("summary") or {} + best_checkpoint = analysis_data.get("best_checkpoint") or {} + best_metric = analysis_data.get("best_metric") or {} + if ( + trainer_state.get("train_framework") == "verl" + and best_checkpoint.get("needs_merge") + and ((trainer_state.get("train_config") or {}).get("result") or {}).get( + "export_huggingface", True + ) + ): + from loopai.skills.Trainer.utils.verl_exporter import export_verl_checkpoint + + export_log_path = str(Path(trainer_state["trainer_output_dir"]) / "model_export.log") + try: + exported_path = export_verl_checkpoint( + str(trainer_state.get("train_output_config_path") or ""), + best_checkpoint, + export_log_path, + app_config={ + "verl_dir": trainer_state.get("verl_dir"), + "verl_env_path": trainer_state.get("verl_env_path"), + "CUDA_VISIBLE_DEVICES": trainer_state.get("CUDA_VISIBLE_DEVICES"), + }, + ) + best_checkpoint["raw_checkpoint_path"] = best_checkpoint.get("path") + best_checkpoint["path"] = exported_path + best_checkpoint["needs_merge"] = False + best_checkpoint["export_log_path"] = export_log_path + summary["best_checkpoint_path"] = exported_path + summary["model_export_status"] = "completed" + except Exception as export_exc: + trainer_state["trainer_model_export_error"] = str(export_exc) + trainer_state["trainer_model_export_log_path"] = export_log_path + summary["model_export_status"] = "failed" + compact_analysis = { + "ok": analysis_payload.get("ok"), + "status": analysis_payload.get("status"), + "message": analysis_payload.get("message"), + "summary": summary, + "best_checkpoint": best_checkpoint, + "best_metric": best_metric, + "model_export_error": trainer_state.get("trainer_model_export_error"), + "error": analysis_payload.get("error"), + } + trainer_state["trainer_result_analysis"] = compact_analysis + trainer_state["trainer_result_analysis_version_id"] = version_id + trainer_state["trainer_result_summary"] = summary + trainer_state["trainer_best_checkpoint"] = best_checkpoint + trainer_state["trainer_best_metric"] = best_metric + if best_checkpoint.get("path") and not best_checkpoint.get("needs_merge"): + trainer_state["trainer_best_checkpoint_path"] = best_checkpoint["path"] + trainer_state["update_model_path"] = best_checkpoint["path"] + if analysis_data.get("checkpoints") and not trainer_state.get("training_checkpoints"): + trainer_state["training_checkpoints"] = [ + item.get("name") for item in analysis_data["checkpoints"] if item.get("name") + ] + if analysis_data.get("metric_records") and not trainer_state.get("training_step_losses"): + trainer_state["training_step_losses"] = analysis_data["metric_records"] + except Exception as exc: + trainer_state["trainer_result_analysis"] = { + "ok": False, + "status": "failed", + "message": "Trainer result analysis failed.", + "error": { + "type": type(exc).__name__, + "detail": str(exc), + }, + } + trainer_state["trainer_result_analysis_version_id"] = version_id + return result + + def run_trainer_standalone( state: Optional[Dict[str, Any]] = None, thread_id: Optional[str] = None, @@ -239,7 +345,9 @@ def run_trainer_standalone( fallback_state = { "task_id": fallback_task_id, "output_dir": fallback_output_root, - "trainer": dict((state or {}).get("trainer") or {}) if isinstance(state, dict) else {}, + "trainer": strip_retired_tracking_fields( + dict((state or {}).get("trainer") or {}) if isinstance(state, dict) else {} + ), } fallback_writer = prepare_trainer_run( fallback_state, @@ -331,6 +439,11 @@ def run_trainer_standalone( ) trainer_state["train_output_config_path"] = prepared_config["config_path"] trainer_state["train_config"] = prepared_config["config"] + # Carry the exact approved bytes into the trusted worker request. + # The worker materializes this snapshot instead of rereading a + # user-editable source path after the approval digest check. + trainer_state["_trainer_approved_config_yaml"] = prepared_config["config_yaml"] + trainer_state["_trainer_approved_config_sha256"] = actual_digest trainer_state["trainer_config_generation_success"] = True trainer_state["_trainer_use_prepared_config"] = True except Exception as exc: @@ -348,6 +461,13 @@ def run_trainer_standalone( _update_trainer_task_state(runtime, trainer_state) raise + if not prepare_only: + trainer_state["_trainer_worker_runtime"] = { + "task_state_loaded": runtime.get("task_state_loaded", False), + "thread_id": runtime.get("thread_id"), + "db_path": runtime.get("db_path"), + } + from loopai.memory import checkpointer, store from loopai.skills.Trainer.trainer_agent import TrainerAgent @@ -379,6 +499,9 @@ def run_trainer_standalone( trainer_state = result.setdefault("trainer", {}) if isinstance(result, dict) else {} trainer_state.pop("_trainer_prepare_only", None) trainer_state.pop("_trainer_use_prepared_config", None) + trainer_state.pop("_trainer_worker_runtime", None) + trainer_state.pop("_trainer_approved_config_yaml", None) + trainer_state.pop("_trainer_approved_config_sha256", None) if prepare_only: try: @@ -421,50 +544,44 @@ def run_trainer_standalone( return result if isinstance(result, dict): - try: - from loopai.skills.Trainer.results import analyze_results - - analysis_payload = analyze_results( - task_id=runtime["thread_id"], - output_dir=resolved_state.get("output_dir", "./outputs"), - trainer_state=trainer_state, + finalize_trainer_result_state(result, task_id=runtime["thread_id"]) + + training_success = trainer_state.get("trainer_training_success") is True + if not training_success: + existing_payload = trainer_state.get("trainer_result") + if isinstance(existing_payload, dict) and existing_payload.get("ok") is False: + payload = existing_payload + event_writer.set_failed(payload) + if emit_result: + print(json.dumps(payload, ensure_ascii=False)) + else: + final_status = trainer_state.get("trainer_training_final_status") or {} + terminal_status = str(final_status.get("status") or "failed").lower() + error_detail = ( + trainer_state.get("train_output_training_error") + or final_status.get("error_message") + or f"training finished with status: {terminal_status}" ) - analysis_data = analysis_payload.get("data") or {} - summary = analysis_data.get("summary") or {} - best_checkpoint = analysis_data.get("best_checkpoint") or {} - best_metric = analysis_data.get("best_metric") or {} - compact_analysis = { - "ok": analysis_payload.get("ok"), - "status": analysis_payload.get("status"), - "message": analysis_payload.get("message"), - "summary": summary, - "best_checkpoint": best_checkpoint, - "best_metric": best_metric, - "error": analysis_payload.get("error"), - } - trainer_state["trainer_result_analysis"] = compact_analysis - trainer_state["trainer_result_summary"] = summary - trainer_state["trainer_best_checkpoint"] = best_checkpoint - trainer_state["trainer_best_metric"] = best_metric - if best_checkpoint.get("path"): - trainer_state["trainer_best_checkpoint_path"] = best_checkpoint["path"] - trainer_state["update_model_path"] = best_checkpoint["path"] - if analysis_data.get("checkpoints") and not trainer_state.get("training_checkpoints"): - trainer_state["training_checkpoints"] = [ - item.get("name") for item in analysis_data["checkpoints"] if item.get("name") - ] - if analysis_data.get("metric_records") and not trainer_state.get("training_step_losses"): - trainer_state["training_step_losses"] = analysis_data["metric_records"] - except Exception as exc: - trainer_state["trainer_result_analysis"] = { - "ok": False, - "status": "failed", - "message": "Trainer result analysis failed.", - "error": { - "type": type(exc).__name__, - "detail": str(exc), - }, - } + payload = emit_error( + RuntimeError(str(error_detail)), + code=( + ErrorCode.INTERRUPTED + if terminal_status == "cancelled" + else ErrorCode.UNHANDLED_EXCEPTION + ), + recoverable=terminal_status != "cancelled", + stream_writer=event_writer, + message="Trainer skill failed.", + exit_process=False, + print_payload=emit_result, + ) + trainer_state["trainer_result"] = payload + trainer_state["trainer_last_error"] = payload["error"] + trainer_state["trainer_event_log_path"] = str(event_writer.event_path) + _update_trainer_task_state(runtime, trainer_state) + if emit_result: + raise SystemExit(1) + return result success_data = { "task_id": runtime["thread_id"], @@ -478,12 +595,14 @@ def run_trainer_standalone( "train_output_training_report_path": trainer_state.get("train_output_training_report_path"), "trainer_result_summary": trainer_state.get("trainer_result_summary"), "trainer_best_checkpoint": trainer_state.get("trainer_best_checkpoint"), + "trainer_model_export_error": trainer_state.get("trainer_model_export_error"), + "trainer_model_export_log_path": trainer_state.get("trainer_model_export_log_path"), } payload = emit_success( data=success_data, message="Trainer skill completed.", stream_writer=event_writer, - exit_process=emit_result, + exit_process=False, print_payload=emit_result, ) if isinstance(result, dict): @@ -492,4 +611,7 @@ def run_trainer_standalone( result.setdefault("trainer", {})["trainer_event_log_path"] = str(event_writer.event_path) _update_trainer_task_state(runtime, result.setdefault("trainer", {})) + if emit_result: + raise SystemExit(0) + return result diff --git a/loopai/skills/Trainer/runtime_config.py b/loopai/skills/Trainer/runtime_config.py index 7b1d57f0..0636b69e 100644 --- a/loopai/skills/Trainer/runtime_config.py +++ b/loopai/skills/Trainer/runtime_config.py @@ -6,19 +6,29 @@ from pathlib import Path from typing import Any, Dict, Optional +from loopai.common.tracking import is_retired_tracking_key, strip_retired_tracking_fields +from loopai.skills.Trainer.rewards import normalize_reward_preset + _DEFAULT_OUTPUT_DIR = "./outputs" _DEFAULT_THREAD_ID = "trainer-default" _DEFAULT_TRAIN_FRAMEWORK = "llamafactory" +_DEFAULT_TRAIN_STAGE = "sft" _DEFAULT_CUDA_VISIBLE_DEVICES = "0" _DEFAULT_TEMPLATE_PATH = ( Path(__file__).resolve().parent / "templates" / "qwen2_5_coder_bird_full_sft.yaml" ) +_DEFAULT_VERL_GRPO_TEMPLATE_PATH = ( + Path(__file__).resolve().parent + / "templates" + / "verl_grpo.yaml" +) _REQUIRED_TRAINER_FIELDS = ( "train_framework", + "train_stage", "train_input_dataset_path", "train_input_task_description", "train_input_config_template_path", @@ -30,7 +40,13 @@ "required": True, "source": "auto", "default": _DEFAULT_TRAIN_FRAMEWORK, - "description": "Training backend. Use llamafactory for SFT by default.", + "description": "Training backend: llamafactory for SFT or verl for GRPO.", + }, + "train_stage": { + "required": True, + "source": "auto", + "default": _DEFAULT_TRAIN_STAGE, + "description": "Training stage. Supported values are sft and grpo.", }, "train_input_dataset_path": { "required": True, @@ -54,10 +70,10 @@ "required": True, "source": "auto", "default": str(_DEFAULT_TEMPLATE_PATH), - "description": "LLaMA-Factory YAML template. The default SFT template is used when omitted.", + "description": "Backend-specific YAML template; Trainer selects the SFT or GRPO default when omitted.", }, "llamafactory_dir": { - "required": True, + "required": False, "source": "user_or_system", "description": "Local LLaMA-Factory repository directory.", "example": "/path/to/LLaMA-Factory/", @@ -68,17 +84,66 @@ "description": "Python environment bin directory for LLaMA-Factory.", "example": "/path/to/miniconda3/envs/llamafactory/bin/", }, + "verl_dir": { + "required": False, + "source": "user_or_system", + "description": "Local verl repository directory, required for GRPO.", + "example": "/path/to/verl/", + }, + "verl_env_path": { + "required": False, + "source": "user_or_system", + "default": "verl", + "description": "Conda environment name or Python environment path used by verl.", + "example": "verl", + }, + "train_input_eval_dataset_path": { + "required": False, + "source": "user", + "description": "Validation parquet required by the initial verl GRPO adapter.", + "example": "/path/to/validation.parquet", + }, + "verl_reward_function_path": { + "required": False, + "source": "user", + "description": "Optional custom reward function Python file for verl GRPO.", + "example": "/path/to/reward.py", + }, + "verl_reward_function_name": { + "required": False, + "source": "auto", + "default": "compute_score", + "description": "Callable name in verl_reward_function_path.", + }, + "verl_reward_mode": { + "required": False, + "source": "auto", + "default": "auto", + "description": "Reward source: auto, preset, or custom.", + }, + "verl_reward_preset": { + "required": False, + "source": "auto", + "default": "auto", + "description": "Named LoopAI reward preset used by auto/preset mode.", + }, + "verl_reward_kwargs": { + "required": False, + "source": "user", + "default": {}, + "description": "Optional keyword arguments passed to the selected reward.", + }, "CUDA_VISIBLE_DEVICES": { "required": False, "source": "auto", "default": _DEFAULT_CUDA_VISIBLE_DEVICES, "description": "CUDA devices used by training. Defaults to single GPU 0.", }, - "train_input_use_swanlab": { + "trainer_persistent_worker": { "required": False, "source": "auto", - "default": False, - "description": "Whether to enable SwanLab logging.", + "default": True, + "description": "Keep training, progress persistence, and finalization alive after the caller disconnects.", }, } @@ -100,6 +165,30 @@ def _as_bool(value: Any, default: bool = False) -> bool: return str(value).strip().lower() in {"1", "true", "yes", "y", "on"} +def _as_int(value: Any, default: int) -> int: + if value is None or value == "": + return default + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _as_dict(value: Any, field_name: str) -> Dict[str, Any]: + if value is None or value == "": + return {} + if isinstance(value, dict): + return copy.deepcopy(value) + if isinstance(value, str): + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise ValueError(f"{field_name} must be a JSON object") from exc + if isinstance(parsed, dict): + return parsed + raise ValueError(f"{field_name} must be a mapping") + + def _unwrap_config_value(value: Any) -> Any: if isinstance(value, dict) and "value" in value: return _unwrap_config_value(value.get("value")) @@ -167,25 +256,60 @@ def _system(state: Dict[str, Any]) -> Dict[str, Any]: return state["system"] -def _existing_default_template_path() -> str: - return str(_DEFAULT_TEMPLATE_PATH) if _DEFAULT_TEMPLATE_PATH.exists() else "" +def _normalize_train_stage(value: Any, framework: str) -> str: + raw = str(value or "").strip().lower() + aliases = { + "rl": "grpo", + "reinforcement_learning": "grpo", + "reinforcement-learning": "grpo", + } + raw = aliases.get(raw, raw) + if not raw: + return "grpo" if framework == "verl" else _DEFAULT_TRAIN_STAGE + if raw not in {"sft", "grpo"}: + raise ValueError(f"unsupported Trainer stage: {raw}; expected sft or grpo") + return raw + + +def _validate_backend_stage(framework: str, stage: str) -> None: + expected = { + "sft": "llamafactory", + "grpo": "verl", + }[stage] + if framework != expected: + raise ValueError( + f"unsupported Trainer backend/stage combination: " + f"train_framework={framework}, train_stage={stage}; " + f"use {expected} for {stage}" + ) + + +def _existing_default_template_path(framework: str = _DEFAULT_TRAIN_FRAMEWORK, stage: str = _DEFAULT_TRAIN_STAGE) -> str: + template_path = _DEFAULT_VERL_GRPO_TEMPLATE_PATH if framework == "verl" and stage == "grpo" else _DEFAULT_TEMPLATE_PATH + return str(template_path) if template_path.exists() else "" def build_trainer_prefill_guide( state: Optional[Dict[str, Any]] = None, *, - task_type: str = "sft", + task_type: str | None = None, ) -> Dict[str, Any]: """Build a guide that tells Codex/user how to complete Trainer config.""" state = copy.deepcopy(state) if isinstance(state, dict) else {"trainer": {}} + state = strip_retired_tracking_fields(state) trainer = _trainer(state) + requested_task = str(task_type or "").strip().lower() + inferred_framework = "verl" if requested_task in {"grpo", "rl", "reinforcement_learning"} else _DEFAULT_TRAIN_FRAMEWORK + framework = str(trainer.get("train_framework") or inferred_framework).strip().lower() + stage = _normalize_train_stage(trainer.get("train_stage") or task_type, framework) + _validate_backend_stage(framework, stage) defaults = { - "train_framework": trainer.get("train_framework") or _DEFAULT_TRAIN_FRAMEWORK, + "train_framework": framework, + "train_stage": stage, "train_input_config_template_path": ( - trainer.get("train_input_config_template_path") or _existing_default_template_path() + trainer.get("train_input_config_template_path") or _existing_default_template_path(framework, stage) ), "CUDA_VISIBLE_DEVICES": trainer.get("CUDA_VISIBLE_DEVICES") or _DEFAULT_CUDA_VISIBLE_DEVICES, - "train_input_use_swanlab": _as_bool(trainer.get("train_input_use_swanlab"), default=False), } for key, value in defaults.items(): if value is not None and value != "": @@ -206,17 +330,27 @@ def build_trainer_prefill_guide( "minimal_state": { "trainer": { "train_framework": defaults["train_framework"], - "train_input_dataset_path": "/path/to/data.json", + "train_stage": defaults["train_stage"], + "train_input_dataset_path": ( + "/path/to/train.parquet" if stage == "grpo" else "/path/to/data.json" + ), "train_input_model_name": "/path/to/base-model", - "train_input_task_description": "SFT a chat assistant for the target task.", + "train_input_task_description": ( + "Optimize the target task with GRPO." if stage == "grpo" + else "SFT a chat assistant for the target task." + ), "train_input_config_template_path": defaults["train_input_config_template_path"], - "llamafactory_dir": "/path/to/LLaMA-Factory/", "CUDA_VISIBLE_DEVICES": defaults["CUDA_VISIBLE_DEVICES"], + **( + {"verl_dir": "/path/to/verl/"} + if stage == "grpo" + else {"llamafactory_dir": "/path/to/LLaMA-Factory/"} + ), } } } return { - "task_type": task_type, + "task_type": stage, "ready": not missing_fields, "missing_fields": missing_fields, "user_required_fields": user_required_fields, @@ -245,6 +379,11 @@ def resolve_trainer_runtime_config( else: state = copy.deepcopy(state) + # Existing databases and caller-provided states may still contain the + # retired tracker secret. Remove it before the state can reach a worker, + # event, API response, or pickle payload. + state = strip_retired_tracking_fields(state) + trainer = _trainer(state) system = _system(state) @@ -265,7 +404,7 @@ def resolve_trainer_runtime_config( trainer.update({ key: value for key, value in db_trainer_config.items() - if value is not None and value != "" + if value is not None and value != "" and not is_retired_tracking_key(key) }) if db_path: state["DB_PATH"] = str(db_path) @@ -292,12 +431,27 @@ def resolve_trainer_runtime_config( kwargs.get("prompt_template_dir") or state.get("prompt_template_dir") or "./loopai/common/prompts", ) - trainer["train_framework"] = _first_non_empty( + trainer["train_framework"] = str(_first_non_empty( kwargs.get("train_framework"), os.getenv("TRAIN_FRAMEWORK"), trainer.get("train_framework"), _DEFAULT_TRAIN_FRAMEWORK, + )).strip().lower() + if trainer["train_framework"] not in {"llamafactory", "verl"}: + raise ValueError( + f"unsupported Trainer framework: {trainer['train_framework']}; " + "expected llamafactory or verl" + ) + trainer["train_stage"] = _normalize_train_stage( + _first_non_empty( + kwargs.get("train_stage"), + kwargs.get("task_type"), + os.getenv("TRAIN_STAGE"), + trainer.get("train_stage"), + ), + trainer["train_framework"], ) + _validate_backend_stage(trainer["train_framework"], trainer["train_stage"]) trainer["train_input_dataset_path"] = _first_non_empty( kwargs.get("train_input_dataset_path"), kwargs.get("dataset_path"), @@ -316,12 +470,18 @@ def resolve_trainer_runtime_config( os.getenv("TRAIN_TASK_DESCRIPTION"), trainer.get("train_input_task_description"), ) + trainer["train_input_eval_dataset_path"] = _first_non_empty( + kwargs.get("train_input_eval_dataset_path"), + kwargs.get("eval_dataset_path"), + os.getenv("TRAIN_EVAL_DATASET_PATH"), + trainer.get("train_input_eval_dataset_path"), + ) trainer["train_input_config_template_path"] = _first_non_empty( kwargs.get("train_input_config_template_path"), kwargs.get("config_template_path"), os.getenv("TRAIN_CONFIG_TEMPLATE_PATH"), trainer.get("train_input_config_template_path"), - _existing_default_template_path(), + _existing_default_template_path(trainer["train_framework"], trainer["train_stage"]), ) trainer["llamafactory_dir"] = _first_non_empty( kwargs.get("llamafactory_dir"), @@ -335,6 +495,115 @@ def resolve_trainer_runtime_config( trainer.get("llamafactory_env_path"), system.get("llamafactory_env_path"), ) + trainer["verl_dir"] = _first_non_empty( + kwargs.get("verl_dir"), + os.getenv("VERL_DIR"), + trainer.get("verl_dir"), + system.get("verl_dir"), + ) + trainer["verl_env_path"] = _first_non_empty( + kwargs.get("verl_env_path"), + kwargs.get("verl_conda_env"), + os.getenv("VERL_ENV_PATH"), + os.getenv("VERL_CONDA_ENV"), + trainer.get("verl_env_path"), + system.get("verl_env_path"), + "verl", + ) + trainer["verl_algorithm"] = str(_first_non_empty( + kwargs.get("verl_algorithm"), + os.getenv("VERL_ALGORITHM"), + trainer.get("verl_algorithm"), + "grpo", + )).strip().lower() + if trainer["train_stage"] == "grpo" and trainer["verl_algorithm"] != "grpo": + raise ValueError("the initial verl integration only supports verl_algorithm=grpo") + trainer["verl_entrypoint"] = str(_first_non_empty( + kwargs.get("verl_entrypoint"), + os.getenv("VERL_ENTRYPOINT"), + trainer.get("verl_entrypoint"), + "verl.trainer.main_ppo", + )) + trainer["verl_rollout_backend"] = str(_first_non_empty( + kwargs.get("verl_rollout_backend"), + os.getenv("VERL_ROLLOUT_BACKEND"), + trainer.get("verl_rollout_backend"), + "vllm", + )).strip().lower() + if trainer["verl_rollout_backend"] not in {"vllm", "sglang"}: + raise ValueError("verl_rollout_backend must be vllm or sglang") + trainer["verl_model_backend"] = str(_first_non_empty( + kwargs.get("verl_model_backend"), + os.getenv("VERL_MODEL_BACKEND"), + trainer.get("verl_model_backend"), + "fsdp", + )).strip().lower() + if trainer["verl_model_backend"] != "fsdp": + raise ValueError("the initial verl GRPO integration only supports verl_model_backend=fsdp") + trainer["verl_reward_function_path"] = _first_non_empty( + kwargs.get("verl_reward_function_path"), + os.getenv("VERL_REWARD_FUNCTION_PATH"), + trainer.get("verl_reward_function_path"), + ) + trainer["verl_reward_function_name"] = str(_first_non_empty( + kwargs.get("verl_reward_function_name"), + os.getenv("VERL_REWARD_FUNCTION_NAME"), + trainer.get("verl_reward_function_name"), + "compute_score", + )) + raw_reward_mode = _first_non_empty( + kwargs.get("verl_reward_mode"), + os.getenv("VERL_REWARD_MODE"), + trainer.get("verl_reward_mode"), + ) + # A path without the new mode field is the legacy custom-reward contract. + if not raw_reward_mode: + raw_reward_mode = "custom" if trainer.get("verl_reward_function_path") else "auto" + trainer["verl_reward_mode"] = str(raw_reward_mode).strip().lower() + if trainer["verl_reward_mode"] not in {"auto", "preset", "custom"}: + raise ValueError("verl_reward_mode must be auto, preset, or custom") + + if trainer["verl_reward_mode"] == "auto": + trainer["verl_reward_preset"] = "auto" + elif trainer["verl_reward_mode"] == "preset": + trainer["verl_reward_preset"] = normalize_reward_preset(_first_non_empty( + kwargs.get("verl_reward_preset"), + os.getenv("VERL_REWARD_PRESET"), + trainer.get("verl_reward_preset"), + "auto", + )) + else: + trainer["verl_reward_preset"] = "" + trainer["verl_reward_kwargs"] = _as_dict( + _first_non_empty( + kwargs.get("verl_reward_kwargs"), + os.getenv("VERL_REWARD_KWARGS"), + trainer.get("verl_reward_kwargs"), + ), + "verl_reward_kwargs", + ) + trainer["verl_selection_metric"] = str(_first_non_empty( + kwargs.get("verl_selection_metric"), + os.getenv("VERL_SELECTION_METRIC"), + trainer.get("verl_selection_metric"), + "val-core/*/acc/mean@*", + )) + trainer["verl_selection_mode"] = str(_first_non_empty( + kwargs.get("verl_selection_mode"), + os.getenv("VERL_SELECTION_MODE"), + trainer.get("verl_selection_mode"), + "max", + )).strip().lower() + if trainer["verl_selection_mode"] not in {"max", "min"}: + raise ValueError("verl_selection_mode must be max or min") + trainer["verl_max_actor_ckpt_to_keep"] = _as_int( + _first_non_empty( + kwargs.get("verl_max_actor_ckpt_to_keep"), + os.getenv("VERL_MAX_ACTOR_CKPT_TO_KEEP"), + trainer.get("verl_max_actor_ckpt_to_keep"), + ), + default=10, + ) trainer["CUDA_VISIBLE_DEVICES"] = str( _first_non_empty( kwargs.get("cuda_visible_devices"), @@ -345,31 +614,18 @@ def resolve_trainer_runtime_config( _DEFAULT_CUDA_VISIBLE_DEVICES, ) ) - trainer["swanlab_api_key"] = _first_non_empty( - kwargs.get("swanlab_api_key"), - os.getenv("SWANLAB_API_KEY"), - trainer.get("swanlab_api_key"), - system.get("swanlab_api_key"), - "", - ) - trainer["train_input_use_swanlab"] = _as_bool( + trainer["trainer_persistent_worker"] = _as_bool( _first_non_empty( - kwargs.get("train_input_use_swanlab"), - kwargs.get("use_swanlab"), - os.getenv("TRAIN_USE_SWANLAB"), - trainer.get("train_input_use_swanlab"), + kwargs.get("trainer_persistent_worker"), + os.getenv("TRAINER_PERSISTENT_WORKER"), + trainer.get("trainer_persistent_worker"), + system.get("trainer_persistent_worker"), ), - default=False, + default=True, ) - if kwargs.get("train_input_swanlab_project") or kwargs.get("swanlab_project"): - trainer["train_input_swanlab_project"] = _first_non_empty( - kwargs.get("train_input_swanlab_project"), - kwargs.get("swanlab_project"), - ) - prefill_guide = build_trainer_prefill_guide( state, - task_type=str(kwargs.get("task_type") or trainer.get("task_type") or "sft"), + task_type=trainer["train_stage"], ) trainer["trainer_missing_fields"] = prefill_guide["missing_fields"] trainer["trainer_prefill_guide"] = prefill_guide @@ -389,4 +645,14 @@ def get_missing_trainer_fields(state: Dict[str, Any]) -> list[str]: missing = [field for field in _REQUIRED_TRAINER_FIELDS if not trainer.get(field)] if trainer.get("train_framework") == "llamafactory" and not trainer.get("llamafactory_dir"): missing.append("llamafactory_dir") + if trainer.get("train_framework") == "verl" and not trainer.get("verl_dir"): + missing.append("verl_dir") + if trainer.get("train_framework") == "verl" and not trainer.get("train_input_eval_dataset_path"): + missing.append("train_input_eval_dataset_path") + if ( + trainer.get("train_framework") == "verl" + and trainer.get("verl_reward_mode") == "custom" + and not trainer.get("verl_reward_function_path") + ): + missing.append("verl_reward_function_path") return missing diff --git a/loopai/skills/Trainer/templates/default_config.json b/loopai/skills/Trainer/templates/default_config.json index a0ae8450..d4a35290 100644 --- a/loopai/skills/Trainer/templates/default_config.json +++ b/loopai/skills/Trainer/templates/default_config.json @@ -36,6 +36,6 @@ "ddp_timeout": 180000000, "include_num_input_tokens_seen": true, "plot_loss": true, - "report_to": ["swanlab"], + "report_to": "none", "run_name": "llamafactory_training" } diff --git a/loopai/agents/Trainer/templates/qwen2.5_sft_single_gpu.yaml b/loopai/skills/Trainer/templates/qwen2.5_sft_single_gpu.yaml similarity index 95% rename from loopai/agents/Trainer/templates/qwen2.5_sft_single_gpu.yaml rename to loopai/skills/Trainer/templates/qwen2.5_sft_single_gpu.yaml index 3aaa03db..15ae5b1d 100644 --- a/loopai/agents/Trainer/templates/qwen2.5_sft_single_gpu.yaml +++ b/loopai/skills/Trainer/templates/qwen2.5_sft_single_gpu.yaml @@ -20,12 +20,11 @@ dataloader_num_workers: 2 output_dir: ./outputs logging_steps: 10 save_steps: 100 +save_total_limit: 10 plot_loss: true overwrite_output_dir: true save_only_model: false report_to: none -use_swanlab: true -swanlab_mode: local ### train per_device_train_batch_size: 4 diff --git a/loopai/skills/Trainer/templates/qwen2_5_coder_bird_full_sft.yaml b/loopai/skills/Trainer/templates/qwen2_5_coder_bird_full_sft.yaml index 77a7551c..812c9aed 100644 --- a/loopai/skills/Trainer/templates/qwen2_5_coder_bird_full_sft.yaml +++ b/loopai/skills/Trainer/templates/qwen2_5_coder_bird_full_sft.yaml @@ -25,9 +25,7 @@ save_total_limit: 10 plot_loss: true overwrite_output_dir: true save_only_model: false -report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow] -use_swanlab: true -swanlab_mode: local +report_to: none ### train per_device_train_batch_size: 1 diff --git a/loopai/skills/Trainer/templates/verl_grpo.yaml b/loopai/skills/Trainer/templates/verl_grpo.yaml new file mode 100644 index 00000000..242b78e4 --- /dev/null +++ b/loopai/skills/Trainer/templates/verl_grpo.yaml @@ -0,0 +1,66 @@ +# LoopAI Verl GRPO launch specification. +# Trainer generates a task-scoped copy and asks for user approval before execution. + +framework: verl +stage: grpo +entrypoint: verl.trainer.main_ppo + +environment: + verl_dir: "" + verl_env_path: "" + cuda_visible_devices: "0" + metrics_jsonl: "" + +# Flat Hydra overrides passed to the selected Verl entrypoint. +overrides: + algorithm.adv_estimator: grpo + algorithm.use_kl_in_reward: false + data.train_files: [] + data.val_files: [] + data.train_batch_size: 128 + data.max_prompt_length: 2048 + data.max_response_length: 2048 + data.filter_overlong_prompts: true + data.truncation: error + actor_rollout_ref.model.path: "" + actor_rollout_ref.model.use_remove_padding: true + actor_rollout_ref.model.enable_gradient_checkpointing: true + actor_rollout_ref.actor.optim.lr: 1.0e-6 + actor_rollout_ref.actor.ppo_mini_batch_size: 64 + actor_rollout_ref.actor.use_dynamic_bsz: true + actor_rollout_ref.actor.ppo_max_token_len_per_gpu: 24576 + actor_rollout_ref.actor.use_kl_loss: true + actor_rollout_ref.actor.kl_loss_coef: 0.001 + actor_rollout_ref.actor.kl_loss_type: low_var_kl + actor_rollout_ref.actor.checkpoint.save_contents: + - model + - optimizer + - extra + actor_rollout_ref.rollout.name: vllm + actor_rollout_ref.rollout.n: 4 + actor_rollout_ref.rollout.tensor_model_parallel_size: 1 + actor_rollout_ref.rollout.gpu_memory_utilization: 0.6 + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz: true + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu: 24576 + actor_rollout_ref.ref.log_prob_use_dynamic_bsz: true + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu: 24576 + actor_rollout_ref.ref.fsdp_config.param_offload: true + trainer.project_name: loopai-grpo + trainer.experiment_name: loopai-grpo + trainer.default_local_dir: "" + trainer.n_gpus_per_node: 8 + trainer.nnodes: 1 + trainer.save_freq: 2 + trainer.test_freq: 2 + trainer.total_epochs: 1 + trainer.max_actor_ckpt_to_keep: 10 + trainer.logger: + - console + - file + reward.custom_reward_function.path: null + reward.custom_reward_function.name: compute_score + +result: + selection_metric: val-core/*/acc/mean@* + selection_mode: max + export_huggingface: true diff --git a/loopai/skills/Trainer/templates/verl_grpo_smoke.yaml b/loopai/skills/Trainer/templates/verl_grpo_smoke.yaml new file mode 100644 index 00000000..9d65e21f --- /dev/null +++ b/loopai/skills/Trainer/templates/verl_grpo_smoke.yaml @@ -0,0 +1,74 @@ +# Fast, task-scoped Verl GRPO smoke test. +# This preset validates data loading, repeated rollout/reward/actor updates, +# and per-step validation without changing the production GRPO defaults. + +framework: verl +stage: grpo +entrypoint: verl.trainer.main_ppo + +environment: + verl_dir: "" + verl_env_path: "" + cuda_visible_devices: "0,1,2,3,4,5,6,7" + metrics_jsonl: "" + +overrides: + algorithm.adv_estimator: grpo + algorithm.use_kl_in_reward: false + data.train_files: [] + data.val_files: [] + data.train_batch_size: 8 + data.train_max_samples: 80 + data.val_max_samples: 8 + data.dataloader_num_workers: 1 + data.max_prompt_length: 1024 + data.max_response_length: 256 + data.filter_overlong_prompts: true + data.truncation: error + actor_rollout_ref.model.path: "" + actor_rollout_ref.model.use_remove_padding: true + actor_rollout_ref.model.enable_gradient_checkpointing: false + actor_rollout_ref.actor.optim.lr: 1.0e-6 + actor_rollout_ref.actor.ppo_mini_batch_size: 8 + actor_rollout_ref.actor.use_dynamic_bsz: true + actor_rollout_ref.actor.use_torch_compile: false + actor_rollout_ref.actor.ppo_max_token_len_per_gpu: 4096 + actor_rollout_ref.actor.use_kl_loss: true + actor_rollout_ref.actor.kl_loss_coef: 0.001 + actor_rollout_ref.actor.kl_loss_type: low_var_kl + actor_rollout_ref.actor.checkpoint.save_contents: + - model + - optimizer + - extra + actor_rollout_ref.rollout.name: vllm + actor_rollout_ref.rollout.n: 2 + actor_rollout_ref.rollout.tensor_model_parallel_size: 1 + actor_rollout_ref.rollout.gpu_memory_utilization: 0.2 + actor_rollout_ref.rollout.enforce_eager: true + actor_rollout_ref.rollout.log_prob_use_dynamic_bsz: true + actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu: 4096 + actor_rollout_ref.ref.log_prob_use_dynamic_bsz: true + actor_rollout_ref.ref.log_prob_max_token_len_per_gpu: 4096 + actor_rollout_ref.ref.fsdp_config.param_offload: false + trainer.project_name: loopai-grpo-smoke + trainer.experiment_name: loopai-grpo-smoke + trainer.default_local_dir: "" + trainer.n_gpus_per_node: 8 + trainer.nnodes: 1 + trainer.val_before_train: false + trainer.total_epochs: 1 + trainer.total_training_steps: 10 + trainer.save_freq: -1 + trainer.test_freq: 1 + trainer.max_actor_ckpt_to_keep: 1 + trainer.logger: + - console + - file + reward.num_workers: 1 + reward.custom_reward_function.path: null + reward.custom_reward_function.name: compute_score + +result: + selection_metric: val-core/*/acc/mean@* + selection_mode: max + export_huggingface: false diff --git a/loopai/skills/Trainer/trainer_agent.py b/loopai/skills/Trainer/trainer_agent.py index 24fe58ae..537282a3 100644 --- a/loopai/skills/Trainer/trainer_agent.py +++ b/loopai/skills/Trainer/trainer_agent.py @@ -250,11 +250,23 @@ def training_execution_node_wrapper(state: LoopAIState) -> LoopAIState: "report_path": result_state.get('trainer', {}).get('train_output_training_report_path') } if not success: - error_payload = build_trainer_error_payload( - RuntimeError(result_state.get('trainer', {}).get('train_output_training_error') or "training execution failed"), - recoverable=True, - message="Trainer training execution failed.", - ) + result_trainer = result_state.get('trainer', {}) + existing_payload = result_trainer.get('trainer_result') + final_status = result_trainer.get('trainer_training_final_status') or {} + terminal_status = str(final_status.get('status') or 'failed').lower() + if isinstance(existing_payload, dict) and existing_payload.get('ok') is False: + error_payload = existing_payload + else: + error_payload = build_trainer_error_payload( + RuntimeError(result_trainer.get('train_output_training_error') or "training execution failed"), + code=( + ErrorCode.INTERRUPTED + if terminal_status == 'cancelled' + else ErrorCode.UNHANDLED_EXCEPTION + ), + recoverable=terminal_status != 'cancelled', + message="Trainer training execution failed.", + ) record_trainer_result(result_state, error_payload) training_error = error_payload["error"] else: @@ -267,7 +279,11 @@ def training_execution_node_wrapper(state: LoopAIState) -> LoopAIState: writer, result_state, "training_execution", - "completed" if success else "failed", + "completed" if success else ( + "cancelled" + if str((result_data.get("final_status") or {}).get("status") or "").lower() == "cancelled" + else "failed" + ), progress=1.0, message=f"训练任务{'成功完成' if success else '执行失败'}", data=result_data, @@ -405,7 +421,6 @@ def get_training_summary(self, state: LoopAIState) -> Dict[str, Any]: "log_path": state.get('trainer', {}).get('train_output_training_log_path'), "report_path": state.get('trainer', {}).get('train_output_training_report_path'), "error": state.get('trainer', {}).get('train_output_training_error'), - "train_output_swanlab_log_path": state.get('trainer', {}).get('train_output_swanlab_log_path') } }, "final_status": "success" if state.get('trainer', {}).get('trainer_training_success', False) else "failed", diff --git a/loopai/skills/Trainer/utils/config_generator.py b/loopai/skills/Trainer/utils/config_generator.py index e9f42ddb..b7b30ccd 100644 --- a/loopai/skills/Trainer/utils/config_generator.py +++ b/loopai/skills/Trainer/utils/config_generator.py @@ -11,6 +11,7 @@ from langchain_openai import ChatOpenAI from loopai.logger import get_logger from loopai.common.prompts import PromptLoader +from loopai.common.tracking import force_local_training_metrics logger = get_logger() @@ -69,8 +70,6 @@ def generate_config( model_name: str = "qwen2.5-7b", output_dir: str = "./output", template_path: Optional[str] = None, - use_swanlab: bool = True, - swanlab_project: str = "llamafactory_training" ) -> Dict[str, Any]: """ 根据任务描述生成训练配置 @@ -81,8 +80,6 @@ def generate_config( model_name: 基础模型名称 output_dir: 输出目录 template_path: 配置模板路径（可选） - use_swanlab: 是否使用 SwanLab 监控 - swanlab_project: SwanLab 项目名称 Returns: 生成的配置字典 @@ -100,8 +97,6 @@ def generate_config( dataset_path=dataset_path, model_name=model_name, output_dir=output_dir, - use_swanlab=use_swanlab, - swanlab_project=swanlab_project ) return config @@ -251,8 +246,6 @@ def _customize_config( dataset_path: str, model_name: str, output_dir: str, - use_swanlab: bool, - swanlab_project: str ) -> Dict[str, Any]: """根据任务描述定制配置""" config = template.copy() @@ -274,10 +267,13 @@ def _customize_config( logger.info("大模型不可用，使用规则式生成配置参数") # 使用原有的规则式方法 config = self._customize_config_with_rules( - config, task_description, use_swanlab, swanlab_project + config, task_description ) - - return config + + # Metrics are parsed from local files. Apply this after every config + # generation path so a template or LLM response cannot re-enable an + # external experiment tracker. + return force_local_training_metrics(config) def _generate_config_with_llm( self, @@ -513,8 +509,6 @@ def _customize_config_with_rules( self, config: Dict[str, Any], task_description: str, - use_swanlab: bool, - swanlab_project: str ) -> Dict[str, Any]: """使用规则式方法定制配置（备用方法）""" @@ -541,12 +535,6 @@ def _customize_config_with_rules( config["lora_r"] = 8 config["lora_alpha"] = 16 config["lora_target"] = "q_proj,v_proj" - # SwanLab 监控设置 (YAML格式) - if use_swanlab: - config["report_to"] = "swanlab" - else: - config["report_to"] = "none" - return config def _get_dataset_name(self, dataset_path: str) -> str: """从数据集路径获取数据集名称""" @@ -586,13 +574,6 @@ def generate_config_explanation(config: Dict[str, Any], task_description: str) - explanation.append(f" LoRA Target: {config.get('lora_target', 'N/A')}") explanation.append("") - report_to = config.get('report_to', 'none') - if report_to == 'swanlab' or (isinstance(report_to, list) and 'swanlab' in report_to): - explanation.append("SwanLab 监控:") - explanation.append(" 已启用SwanLab监控") - explanation.append(f" 日志步数: {config.get('logging_steps', 'N/A')}") - explanation.append("") - explanation.append("配置调整说明:") explanation.append(" - 直接使用YAML模板，避免格式转换") explanation.append(" - 基于qwen2.5-coder配置优化") diff --git a/loopai/skills/Trainer/utils/persistent_worker.py b/loopai/skills/Trainer/utils/persistent_worker.py new file mode 100644 index 00000000..c754b5b1 --- /dev/null +++ b/loopai/skills/Trainer/utils/persistent_worker.py @@ -0,0 +1,335 @@ +from __future__ import annotations + +import json +import os +import pickle +import subprocess +import sys +import time +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Dict, Optional + +from loopai.common.tracking import ( + strip_retired_tracking_environment, + strip_retired_tracking_fields, +) + +try: + import fcntl +except ImportError: # pragma: no cover - Windows compatibility + fcntl = None + + +RUN_STATE_FILENAME = "run_state.json" +WORKER_REQUEST_FILENAME = "worker_request.pkl" +WORKER_RESULT_FILENAME = "worker_result.pkl" +WORKER_LOG_FILENAME = "worker.log" +WORKER_LAUNCH_LOCK_FILENAME = "worker_launch.lock" + + +class PersistentTrainerWorkerError(RuntimeError): + """Raised when a detached Trainer worker cannot finish its request.""" + + +@dataclass +class PersistentWorkerHandle: + run_dir: Path + state_path: Path + result_path: Path + process: Optional[subprocess.Popen] = None + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def worker_paths(run_dir: str | Path) -> Dict[str, Path]: + root = Path(run_dir).expanduser().resolve() + return { + "run_dir": root, + "state": root / RUN_STATE_FILENAME, + "request": root / WORKER_REQUEST_FILENAME, + "result": root / WORKER_RESULT_FILENAME, + "log": root / WORKER_LOG_FILENAME, + "launch_lock": root / WORKER_LAUNCH_LOCK_FILENAME, + } + + +@contextmanager +def _exclusive_launch_lock(path: Path): + """Serialize attach/launch decisions for one Trainer version directory.""" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+", encoding="utf-8") as lock_file: + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def resolve_run_state_path(state: Dict[str, Any]) -> Optional[Path]: + trainer = state.get("trainer") if isinstance(state, dict) else None + if not isinstance(trainer, dict): + return None + run_dir = trainer.get("trainer_output_dir") or trainer.get("output_dir") + if not run_dir: + return None + return worker_paths(str(run_dir))["state"] + + +def _atomic_pickle_dump(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temp_path = path.with_name(f".{path.name}.{os.getpid()}.tmp") + with temp_path.open("wb") as file: + pickle.dump(value, file, protocol=pickle.HIGHEST_PROTOCOL) + file.flush() + os.fsync(file.fileno()) + os.chmod(temp_path, 0o600) + os.replace(temp_path, path) + + +def load_worker_result(path: str | Path) -> Dict[str, Any]: + result_path = Path(path) + with result_path.open("rb") as file: + payload = pickle.load(file) + if not isinstance(payload, dict): + raise PersistentTrainerWorkerError("Trainer worker result is not a mapping") + cleaned = strip_retired_tracking_fields(payload) + if cleaned != payload: + _atomic_pickle_dump(result_path, cleaned) + return cleaned + + +def load_worker_request(path: str | Path) -> Dict[str, Any]: + with Path(path).open("rb") as file: + payload = pickle.load(file) + if not isinstance(payload, dict): + raise PersistentTrainerWorkerError("Trainer worker request is not a mapping") + return strip_retired_tracking_fields(payload) + + +def read_run_state(path: str | Path) -> Dict[str, Any]: + state_path = Path(path) + if not state_path.is_file(): + return {} + try: + value = json.loads(state_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + return value if isinstance(value, dict) else {} + + +def update_run_state( + path: str | Path, + updates: Dict[str, Any], + *, + increment_event_sequence: bool = False, +) -> Dict[str, Any]: + state_path = Path(path) + state_path.parent.mkdir(parents=True, exist_ok=True) + lock_path = state_path.with_suffix(state_path.suffix + ".lock") + + with lock_path.open("a+", encoding="utf-8") as lock_file: + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + current = read_run_state(state_path) + current.update(updates) + if increment_event_sequence: + current["event_sequence"] = int(current.get("event_sequence") or 0) + 1 + current["updated_at"] = utc_now_iso() + + temp_path = state_path.with_name(f".{state_path.name}.{os.getpid()}.tmp") + with temp_path.open("w", encoding="utf-8") as file: + json.dump(current, file, ensure_ascii=False, indent=2, default=str) + file.flush() + os.fsync(file.fileno()) + os.replace(temp_path, state_path) + return current + finally: + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + +def record_worker_event(state: Dict[str, Any], event_payload: Dict[str, Any]) -> None: + state_path = resolve_run_state_path(state) + if state_path is None or not state_path.exists(): + return + data = event_payload.get("data") if isinstance(event_payload, dict) else None + updates: Dict[str, Any] = { + "phase": event_payload.get("node"), + "event_status": event_payload.get("status"), + "progress": event_payload.get("progress"), + "message": event_payload.get("message"), + "last_event": event_payload, + } + if isinstance(data, dict): + for source, target in ( + ("current_step", "current_step"), + ("total_steps", "total_steps"), + ("elapsed_time", "elapsed_time"), + ("training_pid", "training_pid"), + ("process_group_id", "training_process_group_id"), + ): + if data.get(source) is not None: + updates[target] = data[source] + update_run_state(state_path, updates, increment_event_sequence=True) + + +def _pid_is_alive(pid: Any) -> bool: + try: + value = int(pid) + if value <= 0: + return False + os.kill(value, 0) + return True + except (TypeError, ValueError, ProcessLookupError): + return False + except PermissionError: + return True + + +def launch_or_attach_persistent_worker(state: Dict[str, Any]) -> PersistentWorkerHandle: + cleaned_state = strip_retired_tracking_fields(state) + state.clear() + state.update(cleaned_state) + trainer = state.get("trainer") if isinstance(state, dict) else None + if not isinstance(trainer, dict) or not trainer.get("trainer_output_dir"): + raise PersistentTrainerWorkerError("trainer_output_dir is required before launching the worker") + + paths = worker_paths(trainer["trainer_output_dir"]) + paths["run_dir"].mkdir(parents=True, exist_ok=True) + handle = PersistentWorkerHandle( + run_dir=paths["run_dir"], + state_path=paths["state"], + result_path=paths["result"], + ) + + with _exclusive_launch_lock(paths["launch_lock"]): + if paths["result"].is_file(): + return handle + + existing = read_run_state(paths["state"]) + existing_worker_pid = existing.get("worker_pid") + if existing.get("status") in {"queued", "running"} and _pid_is_alive(existing_worker_pid): + return handle + if existing.get("status") in {"queued", "running"} and _pid_is_alive(existing.get("training_pid")): + raise PersistentTrainerWorkerError( + "Trainer worker exited while its training process is still alive; refusing to start a duplicate run" + ) + + trainer["trainer_run_state_path"] = str(paths["state"]) + trainer["trainer_worker_log_path"] = str(paths["log"]) + _atomic_pickle_dump(paths["request"], {"state": state}) + update_run_state( + paths["state"], + { + "status": "queued", + "task_id": state.get("task_id"), + "version_id": trainer.get("trainer_version_id"), + "framework": trainer.get("train_framework"), + "run_dir": str(paths["run_dir"]), + "request_path": str(paths["request"]), + "result_path": str(paths["result"]), + "worker_log_path": str(paths["log"]), + "submitted_at": utc_now_iso(), + }, + ) + + project_root = Path(__file__).resolve().parents[4] + env = strip_retired_tracking_environment(os.environ) + current_pythonpath = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = os.pathsep.join( + item for item in (str(project_root), current_pythonpath) if item + ) + env["LOOPAI_TRAINER_WORKER_CHILD"] = "1" + + with paths["log"].open("ab", buffering=0) as worker_log: + process = subprocess.Popen( + [ + sys.executable, + "-m", + "loopai.skills.Trainer.worker_entry", + "--request", + str(paths["request"]), + ], + stdin=subprocess.DEVNULL, + stdout=worker_log, + stderr=subprocess.STDOUT, + cwd=str(project_root), + env=env, + start_new_session=(os.name == "posix"), + close_fds=True, + ) + handle.process = process + trainer["trainer_worker_pid"] = process.pid + update_run_state( + paths["state"], + { + "worker_pid": process.pid, + "worker_process_group_id": process.pid if os.name == "posix" else None, + }, + ) + return handle + + +def wait_for_persistent_worker( + handle: PersistentWorkerHandle, + *, + stream_writer: Optional[Callable[[Dict[str, Any]], Any]] = None, + poll_interval: float = 1.0, +) -> Dict[str, Any]: + last_sequence = -1 + dead_since: Optional[float] = None + + while True: + if handle.result_path.is_file(): + payload = load_worker_result(handle.result_path) + if handle.process is not None: + try: + handle.process.wait(timeout=5.0) + except subprocess.TimeoutExpired: + pass + if payload.get("ok") is True and isinstance(payload.get("state"), dict): + return strip_retired_tracking_fields(payload["state"]) + error = payload.get("error") or {} + detail = error.get("detail") if isinstance(error, dict) else str(error) + raise PersistentTrainerWorkerError(detail or "Trainer worker failed") + + run_state = read_run_state(handle.state_path) + sequence = int(run_state.get("event_sequence") or 0) + if sequence != last_sequence: + last_sequence = sequence + last_event = run_state.get("last_event") + if stream_writer is not None and isinstance(last_event, dict): + try: + stream_writer(last_event) + except Exception: + pass + + worker_pid = run_state.get("worker_pid") + process_alive = ( + handle.process.poll() is None + if handle.process is not None + else _pid_is_alive(worker_pid) + ) + if process_alive: + dead_since = None + else: + dead_since = dead_since or time.monotonic() + if time.monotonic() - dead_since >= max(2.0, poll_interval * 2): + status = run_state.get("status") or "unknown" + raise PersistentTrainerWorkerError( + f"Trainer worker exited without a result (status={status}, log={handle.run_dir / WORKER_LOG_FILENAME})" + ) + + time.sleep(poll_interval) + + +def write_worker_result(path: str | Path, payload: Dict[str, Any]) -> None: + _atomic_pickle_dump(Path(path), payload) diff --git a/loopai/skills/Trainer/utils/realtime_log_parser.py b/loopai/skills/Trainer/utils/realtime_log_parser.py index ea0d055e..fea74d1c 100644 --- a/loopai/skills/Trainer/utils/realtime_log_parser.py +++ b/loopai/skills/Trainer/utils/realtime_log_parser.py @@ -38,6 +38,11 @@ def __init__(self, total_steps: int = 0, total_epochs: int = 0): # 更多常见的训练指标 self.accuracy_pattern = re.compile(rf'(?:accuracy|acc)[:\s=]+({number_pattern})', re.IGNORECASE) self.perplexity_pattern = re.compile(rf'(?:perplexity|ppl)[:\s=]+({number_pattern})', re.IGNORECASE) + # Verl console logger emits one line in the form: + # step:12 - critic/rewards/mean:0.42 - training/global_step:12 + self.key_value_pattern = re.compile( + rf'(? Optional[float]: if isinstance(value, bool): @@ -61,6 +66,24 @@ def _fill_step_from_epoch(self, metrics: Dict[str, Any]): except (TypeError, ValueError, OverflowError, ZeroDivisionError): return + def _normalize_verl_metrics(self, metrics: Dict[str, Any]) -> None: + if 'training/global_step' in metrics: + metrics['step'] = int(float(metrics['training/global_step'])) + elif 'global_step' in metrics and 'step' not in metrics: + metrics['step'] = int(float(metrics['global_step'])) + if 'training/epoch' in metrics and 'epoch' not in metrics: + metrics['epoch'] = metrics['training/epoch'] + if 'critic/rewards/mean' in metrics: + metrics['reward'] = metrics['critic/rewards/mean'] + if 'actor/pg_loss' in metrics: + metrics['policy_loss'] = metrics['actor/pg_loss'] + val_keys = sorted( + key for key, value in metrics.items() + if key.startswith('val-core/') and self._to_number(value) is not None + ) + if val_keys: + metrics['val_score'] = metrics[val_keys[0]] + def extract_metrics(self, line: str) -> Dict[str, Any]: """从日志行中提取指标""" metrics = {} @@ -87,7 +110,16 @@ def extract_metrics(self, line: str) -> Dict[str, Any]: except json.JSONDecodeError: continue - # 如果没有找到JSON格式，尝试单独提取各个指标 + # Verl emits flat key:value pairs rather than JSON. + if not metrics: + for key, value_str in self.key_value_pattern.findall(line): + try: + number = float(value_str) + metrics[key] = int(number) if key in {'step', 'global_step', 'training/global_step'} else number + except (ValueError, OverflowError): + continue + + # 如果没有找到JSON或 Verl 指标，尝试单独提取各个指标 if not metrics: patterns = { 'loss': self.loss_pattern, @@ -116,6 +148,8 @@ def extract_metrics(self, line: str) -> Dict[str, Any]: self._fill_step_from_epoch(metrics) + self._normalize_verl_metrics(metrics) + return metrics @@ -145,7 +179,11 @@ def _find_total_steps(self) -> Optional[int]: with open(self.log_path, 'r', encoding='utf-8', errors='ignore') as f: for line in f: - step_match = re.search(r'Total optimization steps = ([\d,]+)', line) + step_match = re.search( + r'(?:Total optimization steps|total_training_steps|total steps)\s*[:=]\s*([\d,]+)', + line, + re.IGNORECASE, + ) epoch_match = re.search(r'Num Epochs = ([\d,]+)', line) if step_match: self.total_steps = int(step_match.group(1).replace(',', '')) @@ -221,6 +259,10 @@ def _monitoring_loop(self): """监控循环""" while self.running: try: + if not self.if_total_steps_recorded: + self._find_total_steps() + self.extractor.total_steps = self.total_steps + self.extractor.total_epochs = self.total_epochs new_metrics = self._parse_new_lines() if new_metrics: self._save_metrics(new_metrics) @@ -237,9 +279,9 @@ def start_monitoring(self): return self.running = True - while not self.if_total_steps_recorded: - self._find_total_steps() - time.sleep(1) + # Do not block process startup while waiting for a framework-specific + # total-step log line. The monitor updates it later when available. + self._find_total_steps() self._initialize_metrics_file() self.extractor = MetricsExtractor(total_steps=self.total_steps, total_epochs=self.total_epochs) self.thread = threading.Thread(target=self._monitoring_loop, daemon=True) @@ -251,6 +293,9 @@ def stop_monitoring(self): self.running = False if self.thread and self.thread.is_alive(): self.thread.join(timeout=5) + final_metrics = self._parse_new_lines() if hasattr(self, 'extractor') else [] + if final_metrics: + self._save_metrics(final_metrics) print("已停止日志监控") def get_latest_metrics(self, count: int = 10) -> List[Dict[str, Any]]: diff --git a/loopai/skills/Trainer/utils/stream_events.py b/loopai/skills/Trainer/utils/stream_events.py index 062aee3e..cd505833 100644 --- a/loopai/skills/Trainer/utils/stream_events.py +++ b/loopai/skills/Trainer/utils/stream_events.py @@ -81,6 +81,7 @@ def prepare_trainer_run( context_id=context_id, log_file_path=output_root, version_id=initial_version_id, + event_dir=get_trainer_event_dir(state), ) if seed_writer.version_id is None: seed_writer.refresh_version_id() @@ -116,6 +117,7 @@ def prepare_trainer_run( context_id=context_id, log_file_path=output_root, version_id=run_id, + event_dir=get_trainer_event_dir(state), ) @@ -173,7 +175,13 @@ def _persist_trainer_event(state: dict[str, Any], event: StreamEvent) -> None: version_id=version_id, ) event.version_id = writer.version_id - writer(event) + persisted_payload = event.json() + if event.status == "completed": + writer.set_completed(persisted_payload) + elif event.status in {"failed", "cancelled"}: + writer.set_failed(persisted_payload) + else: + writer.set_running(persisted_payload) state.setdefault("trainer", {})["trainer_event_log_path"] = str(writer.event_path) except Exception as exc: logger.warning(f"写入 Trainer 事件日志失败: {exc}") @@ -224,5 +232,11 @@ def emit_trainer_event( ) _persist_trainer_event(state, event) payload = event.json() + try: + from loopai.skills.Trainer.utils.persistent_worker import record_worker_event + + record_worker_event(state, payload) + except Exception as exc: + logger.warning(f"更新 Trainer Worker 状态失败: {exc}") writer(payload) return event diff --git a/loopai/skills/Trainer/utils/task_manager.py b/loopai/skills/Trainer/utils/task_manager.py index 04e18a58..d9bf1e0d 100644 --- a/loopai/skills/Trainer/utils/task_manager.py +++ b/loopai/skills/Trainer/utils/task_manager.py @@ -12,17 +12,23 @@ import signal import subprocess import shutil -from datetime import datetime -from typing import Dict, Optional, List +from pathlib import Path +from typing import Dict, Optional import threading from concurrent.futures import ThreadPoolExecutor import json import yaml from loopai.logger import get_logger +from loopai.common.tracking import ( + assert_no_retired_tracking, + contains_retired_tracking_reference, + strip_retired_tracking_environment, +) from .task_status import TaskStatus from .task_tools import ensure_directory_exists, get_current_timestamp from .realtime_log_parser import RealTimeLogParser +from .verl_launcher import build_verl_launch logger = get_logger() @@ -60,7 +66,6 @@ def __init__(self, configs_dir: str, logs_dir: str, runs_dir: str, app_config: d - verl_dir: verl 安装目录 - llamafactory_env_path: LlamaFactory 虚拟环境路径 - CUDA_VISIBLE_DEVICES: GPU 设备号 - - swanlab_api_key: SwanLab API 密钥 """ self.configs_dir = configs_dir self.logs_dir = logs_dir @@ -133,7 +138,7 @@ def start_training(self, task_id: str, output_dir: str) -> bool: def _get_safe_env(self) -> dict: """获取安全的环境变量配置""" - env = os.environ.copy() + env = strip_retired_tracking_environment(os.environ) # 从配置中获取 env["CUDA_VISIBLE_DEVICES"] = self.app_config.get("CUDA_VISIBLE_DEVICES", "0,1") @@ -146,13 +151,6 @@ def _get_safe_env(self) -> dict: else: logger.warning("未找到LLAMAFACTORY_ENV_PATH配置，将使用系统默认的llamafactory-cli") - # 检查必需的API密钥 - swanlab_key = self.app_config.get("swanlab_api_key", "") - if swanlab_key: - env["SWANLAB_API_KEY"] = swanlab_key - else: - logger.warning("未找到SWANLAB_API_KEY配置，某些功能可能无法正常工作") - return env def _run_training(self, task_id: str) -> None: @@ -198,6 +196,12 @@ def _run_training(self, task_id: str) -> None: def _run_llamafactory_training(self, task_info: dict, config_path: str, log_path: str, log_parser: RealTimeLogParser) -> None: """执行 LlamaFactory 训练""" + with open(config_path, "r", encoding="utf-8") as config_file: + config = yaml.safe_load(config_file) or {} + if not isinstance(config, dict): + raise ValueError(f"LLaMA-Factory config must be a mapping: {config_path}") + assert_no_retired_tracking(config) + env = self._get_safe_env() config_env_path = self.app_config.get("llamafactory_env_path", "") env_path = env.get("LLAMAFACTORY_ENV_PATH") or config_env_path @@ -295,20 +299,29 @@ def _run_llamafactory_training(self, task_info: dict, config_path: str, log_path def _run_verl_training(self, task_info: dict, config_path: str, log_path: str, log_parser: RealTimeLogParser) -> None: """执行 verl 训练""" - env = self._get_safe_env() - - # verl 环境配置 - 可通过 app_config 传入 - verl_env_path = self.app_config.get("verl_env_path", "") - if verl_env_path: - env["PYTHONPATH"] = os.path.join(verl_env_path, "lib/python3.10/site-packages") - env["PATH"] = f"{os.path.join(verl_env_path, 'bin')}:{env.get('PATH', '')}" - env["PYTHONNOUSERSITE"] = "True" - - cmd = ["bash", config_path] - logger.info(f"训练命令: {' '.join(cmd)}") - - # 启动实时日志解析 - log_parser.start_monitoring() + suffix = os.path.splitext(config_path)[1].lower() + if suffix in {".yaml", ".yml"}: + cmd, cwd, env = build_verl_launch(config_path, self.app_config) + logger.info( + f"Verl GRPO 命令已由审批 YAML 安全生成: {' '.join(cmd[:3])} " + f"({len(cmd) - 3} 个 Hydra overrides)" + ) + elif suffix == ".sh": + # Compatibility for historical tasks. New Trainer-generated configs + # always use the shell-free YAML path above. + script_text = Path(config_path).read_text(encoding="utf-8", errors="ignore") + if contains_retired_tracking_reference(script_text): + raise ValueError( + "legacy Verl shell config references the retired experiment tracker; " + "prepare a new YAML config before training" + ) + env = self._get_safe_env() + env["PYTHONNOUSERSITE"] = "True" + cmd = ["bash", config_path] + cwd = self.verl_dir + logger.warning("正在执行旧版 Verl shell 配置；新任务应使用审批后的 YAML") + else: + raise ValueError(f"Unsupported Verl config format: {config_path}") with open(log_path, 'w', encoding='utf-8') as log_file: process = subprocess.Popen( @@ -316,7 +329,7 @@ def _run_verl_training(self, task_info: dict, config_path: str, log_path: str, l stdout=log_file, stderr=subprocess.STDOUT, universal_newlines=True, - cwd=self.verl_dir, + cwd=cwd, env=env, start_new_session=(os.name == "posix"), ) @@ -329,6 +342,11 @@ def _run_verl_training(self, task_info: dict, config_path: str, log_path: str, l self._terminate_task_process(task_info) return + try: + log_parser.start_monitoring() + except Exception as exc: + logger.error(f"启动 Verl 日志监控失败: {exc}") + return_code = process.wait() if task_info.get('cancel_requested'): @@ -419,68 +437,6 @@ def cleanup_completed_tasks(self, max_keep: int = 100) -> None: for task_id, _ in tasks_to_remove: del self.tasks[task_id] - def get_train_output_swanlab_log_path(self, task_id: str) -> Optional[str]: - """获取指定任务的SwanLab日志文件夹路径""" - if task_id not in self.tasks: - return None - - task_info = self.tasks[task_id] - swanlog_dir = os.path.join(self.llamafactory_dir, "swanlog") - - if not os.path.exists(swanlog_dir): - return None - - started_at = task_info.get('started_at') - if not started_at: - return None - - try: - task_start_time = datetime.fromisoformat(started_at.replace('Z', '+00:00')) - - log_folders = [] - for item in os.listdir(swanlog_dir): - item_path = os.path.join(swanlog_dir, item) - if os.path.isdir(item_path) and item.startswith('run-'): - folder_create_time = datetime.fromtimestamp(os.path.getctime(item_path)) - - if folder_create_time >= task_start_time: - log_folders.append((item_path, folder_create_time)) - - if log_folders: - log_folders.sort(key=lambda x: x[1]) - return log_folders[-1][0] - - except Exception as e: - logger.error(f"Error finding SwanLab log folder for task {task_id}: {e}") - - return None - - def get_all_swanlab_logs(self) -> List[Dict[str, str]]: - """获取所有SwanLab日志文件夹""" - swanlog_dir = os.path.join(self.llamafactory_dir, "swanlog") - - if not os.path.exists(swanlog_dir): - return [] - - log_folders = [] - try: - for item in os.listdir(swanlog_dir): - item_path = os.path.join(swanlog_dir, item) - if os.path.isdir(item_path) and item.startswith('run-'): - folder_create_time = datetime.fromtimestamp(os.path.getctime(item_path)) - log_folders.append({ - 'folder_name': item, - 'folder_path': item_path, - 'created_at': folder_create_time.isoformat() - }) - - log_folders.sort(key=lambda x: x['created_at'], reverse=True) - - except Exception as e: - logger.error(f"Error listing SwanLab log folders: {e}") - - return log_folders - def get_task_metrics(self, task_id: str, count: int = 100) -> Optional[Dict]: """获取任务的训练指标数据""" if task_id not in self.log_parsers: diff --git a/loopai/skills/Trainer/utils/training_executor.py b/loopai/skills/Trainer/utils/training_executor.py index 35748d27..066f399b 100644 --- a/loopai/skills/Trainer/utils/training_executor.py +++ b/loopai/skills/Trainer/utils/training_executor.py @@ -1,15 +1,19 @@ """ 训练执行工具 -调用 LlamaFactory 进行模型微调并集成 SwanLab 监控 +调用 LlamaFactory 进行模型微调并使用本地日志监控 """ import os import subprocess import sys -import json import time -from pathlib import Path -from typing import Dict, List, Any, Optional +from typing import Dict, List, Any +import yaml + +from loopai.common.tracking import ( + assert_no_retired_tracking, + strip_retired_tracking_environment, +) from loopai.logger import get_logger logger = get_logger() @@ -26,8 +30,6 @@ def execute_training( self, config_path: str, output_dir: str, - use_swanlab: bool = True, - swanlab_project: str = "llamafactory_training" ) -> Dict[str, Any]: """ 执行 LlamaFactory 训练 @@ -35,8 +37,6 @@ def execute_training( Args: config_path: 训练配置文件路径 output_dir: 输出目录 - use_swanlab: 是否使用 SwanLab 监控 - swanlab_project: SwanLab 项目名称 Returns: 训练结果字典 @@ -48,7 +48,6 @@ def execute_training( "config_path": config_path, "output_dir": output_dir, "log_file": None, - "swanlab_url": None, "error_message": None, "training_time": 0 } @@ -58,9 +57,12 @@ def execute_training( if not os.path.exists(config_path): result["error_message"] = f"配置文件不存在: {config_path}" return result - - # 准备环境 - # self._prepare_environment(use_swanlab, swanlab_project) + + with open(config_path, "r", encoding="utf-8") as config_file: + config = yaml.safe_load(config_file) or {} + if not isinstance(config, dict): + raise ValueError(f"training config must be a mapping: {config_path}") + assert_no_retired_tracking(config) # 创建输出目录 os.makedirs(output_dir, exist_ok=True) @@ -84,7 +86,6 @@ def execute_training( if success: result["success"] = True - result["swanlab_url"] = self._get_swanlab_url(swanlab_project) if use_swanlab else None logger.info("训练完成!") else: result["error_message"] = "训练过程中发生错误，请查看日志文件" @@ -154,14 +155,9 @@ def stop_training(self): except Exception as e: logger.error(f"停止训练进程时发生错误: {str(e)}") - def _prepare_environment(self, use_swanlab: bool, swanlab_project: str): + def _prepare_environment(self): """准备训练环境""" - - # 设置环境变量 - if use_swanlab: - os.environ["SWANLAB_PROJECT"] = swanlab_project - os.environ["SWANLAB_EXPERIMENT_NAME"] = f"llamafactory_{int(time.time())}" - + # 检查 LlamaFactory 是否安装 try: import llamafactory @@ -170,15 +166,6 @@ def _prepare_environment(self, use_swanlab: bool, swanlab_project: str): logger.warning("LlamaFactory 未安装，尝试安装...") self._install_llamafactory() - # 检查 SwanLab 是否安装 - if use_swanlab: - try: - import swanlab - logger.info(f"SwanLab 版本: {swanlab.__version__}") - except ImportError: - logger.warning("SwanLab 未安装，尝试安装...") - self._install_swanlab() - def _install_llamafactory(self): """安装 LlamaFactory""" try: @@ -191,17 +178,6 @@ def _install_llamafactory(self): logger.error(f"安装 LlamaFactory 失败: {str(e)}") raise - def _install_swanlab(self): - """安装 SwanLab""" - try: - subprocess.check_call([ - sys.executable, "-m", "pip", "install", "swanlab" - ]) - logger.info("SwanLab 安装成功") - except subprocess.CalledProcessError as e: - logger.error(f"安装 SwanLab 失败: {str(e)}") - raise - def _build_training_command(self, config_path: str, log_file: str) -> List[str]: """构建训练命令""" @@ -222,7 +198,8 @@ def _run_training_process(self, cmd: List[str], log_file: str) -> bool: stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, - bufsize=1 + bufsize=1, + env=strip_retired_tracking_environment(os.environ), ) # 实时写入日志 @@ -266,18 +243,6 @@ def _parse_training_progress(self, log_line: str) -> Dict[str, Any]: return progress - def _get_swanlab_url(self, project_name: str) -> Optional[str]: - """获取 SwanLab 项目 URL""" - - try: - # 这里需要根据 SwanLab 的 API 来获取项目 URL - # 暂时返回默认格式 - return f"https://swanlab.cn/project/{project_name}" - except Exception as e: - logger.warning(f"获取 SwanLab URL 时发生错误: {str(e)}") - return None - - def validate_training_environment() -> Dict[str, Any]: """验证训练环境""" @@ -291,7 +256,7 @@ def validate_training_environment() -> Dict[str, Any]: # 检查必要的包 required_packages = ["torch", "transformers", "datasets"] - optional_packages = ["llamafactory", "swanlab"] + optional_packages = ["llamafactory"] for package in required_packages + optional_packages: try: @@ -340,10 +305,6 @@ def generate_training_report(result: Dict[str, Any]) -> str: if result.get("log_file"): report.append(f"日志文件: {result['log_file']}") - # SwanLab 链接 - if result.get("swanlab_url"): - report.append(f"SwanLab 监控: {result['swanlab_url']}") - # 错误信息 if result.get("error_message"): report.append("") diff --git a/loopai/skills/Trainer/utils/training_log_parser.py b/loopai/skills/Trainer/utils/training_log_parser.py index 6d3d9f1d..f6e7dae9 100644 --- a/loopai/skills/Trainer/utils/training_log_parser.py +++ b/loopai/skills/Trainer/utils/training_log_parser.py @@ -4,6 +4,7 @@ """ import os +import json import re from typing import Optional, Tuple, Dict @@ -18,6 +19,69 @@ def __init__(self): r'\|\s*(\d+)/(\d+)\s*\[(\d{2}:\d{2})<(\d{2}:\d{2}),\s*[\d.]+s/it\]' ) self.total_steps = None # 用于区分训练和评估进度 + self.verl_step_pattern = re.compile(r'(?:^|\s-\s)step:(\d+)') + self.verl_total_pattern = re.compile( + r'(?:total_training_steps|total steps)\s*[:=]\s*([\d,]+)', + re.IGNORECASE, + ) + self.tqdm_step_pattern = re.compile( + r'Training Progress:.*?\|\s*(\d+)/(\d+)\s*\[', + re.IGNORECASE, + ) + + def parse_verl_metrics_progress( + self, + log_path: str, + metrics_path: str, + ) -> Optional[Dict[str, str]]: + """Read Verl's structured JSONL metrics instead of its tqdm output.""" + if not os.path.isfile(metrics_path): + return None + + total_steps = None + if os.path.isfile(log_path): + try: + with open(log_path, "r", encoding="utf-8", errors="ignore") as log_file: + for line in log_file: + for match in self.verl_total_pattern.finditer(line): + total_steps = int(match.group(1).replace(",", "")) + for match in self.tqdm_step_pattern.finditer(line): + total_steps = int(match.group(2)) + except OSError: + total_steps = None + + try: + with open(metrics_path, "r", encoding="utf-8", errors="ignore") as metrics_file: + lines = metrics_file.readlines() + except OSError: + return None + + current_step = None + for line in reversed(lines): + try: + record = json.loads(line) + except (json.JSONDecodeError, TypeError): + continue + if not isinstance(record, dict): + continue + data = record.get("data") if isinstance(record.get("data"), dict) else record + raw_step = data.get("training/global_step", data.get("global_step", record.get("step"))) + try: + current_step = int(float(raw_step)) + except (TypeError, ValueError, OverflowError): + continue + break + + if current_step is None or not total_steps: + return None + return { + "current_step": str(current_step), + "total_steps": str(total_steps), + "elapsed_time": "", + "remaining_time": "", + "progress_text": f"{current_step}/{total_steps}", + "time_text": "Verl metrics", + } def parse_training_progress(self, log_path: str) -> Optional[Dict[str, str]]: """ @@ -47,28 +111,52 @@ def parse_training_progress(self, log_path: str) -> Optional[Dict[str, str]]: with open(log_path, 'r', encoding='utf-8', errors='ignore') as f: lines = f.readlines() - # 从后往前查找进度信息 - for line in reversed(lines): - match = self.progress_pattern.search(line) - if match: - current_step = match.group(1) - total_steps = match.group(2) - elapsed_time = match.group(3) - remaining_time = match.group(4) - - # 如果是第一次解析，记录总步数用于区分训练和评估 - if self.total_steps is None: - self.total_steps = total_steps - - # 只返回与训练总步数匹配的进度（忽略评估进度） - if total_steps == self.total_steps: + # tqdm uses carriage returns, so many refreshes can occupy one + # logical line. Pick the last match for the largest total (the + # training loop) instead of the first match such as 1/58. + progress_matches = [ + match + for line in lines + for match in self.progress_pattern.finditer(line) + ] + if progress_matches: + training_total = max(int(match.group(2)) for match in progress_matches) + match = next( + match + for match in reversed(progress_matches) + if int(match.group(2)) == training_total + ) + current_step = match.group(1) + total_steps = match.group(2) + elapsed_time = match.group(3) + remaining_time = match.group(4) + self.total_steps = total_steps + return { + 'current_step': current_step, + 'total_steps': total_steps, + 'elapsed_time': elapsed_time, + 'remaining_time': remaining_time, + 'progress_text': f"{current_step}/{total_steps}", + 'time_text': f"{elapsed_time}<{remaining_time}" + } + + total_steps = None + for line in lines: + total_match = self.verl_total_pattern.search(line) + if total_match: + total_steps = int(total_match.group(1).replace(',', '')) + if total_steps: + for line in reversed(lines): + step_match = self.verl_step_pattern.search(line) + if step_match: + current_step = int(step_match.group(1)) return { - 'current_step': current_step, - 'total_steps': total_steps, - 'elapsed_time': elapsed_time, - 'remaining_time': remaining_time, + 'current_step': str(current_step), + 'total_steps': str(total_steps), + 'elapsed_time': '', + 'remaining_time': '', 'progress_text': f"{current_step}/{total_steps}", - 'time_text': f"{elapsed_time}<{remaining_time}" + 'time_text': 'Verl GRPO', } except Exception as e: @@ -138,4 +226,4 @@ def get_task_progress_percentage(task_id: str, logs_dir: str = None) -> float: if progress_info: parser = TrainingLogParser() return parser.get_progress_percentage(progress_info) - return 0.0 \ No newline at end of file + return 0.0 diff --git a/loopai/skills/Trainer/utils/verl_config_generator.py b/loopai/skills/Trainer/utils/verl_config_generator.py new file mode 100644 index 00000000..c8e72a04 --- /dev/null +++ b/loopai/skills/Trainer/utils/verl_config_generator.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import copy +from pathlib import Path +from typing import Any, Dict + +import yaml + +from loopai.common.tracking import assert_no_retired_tracking, strip_retired_tracking_config +from loopai.skills.Trainer.rewards import normalize_reward_preset + + +_REWARD_ROUTER_PATH = ( + Path(__file__).resolve().parents[1] + / "rewards" + / "router.py" +) + + +def _gpu_count(cuda_visible_devices: Any) -> int: + devices = [item.strip() for item in str(cuda_visible_devices or "0").split(",") if item.strip()] + return max(1, len(devices)) + + +def generate_verl_grpo_config(state: Dict[str, Any], template_path: str) -> Dict[str, Any]: + """Create the task-scoped, user-approvable Verl GRPO launch YAML.""" + path = Path(template_path).expanduser().resolve() + if not path.is_file(): + raise FileNotFoundError(f"Verl GRPO template does not exist: {path}") + + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + raise ValueError(f"Verl GRPO template must contain a YAML mapping: {path}") + + config = copy.deepcopy(raw) + if config.get("framework") != "verl" or config.get("stage") != "grpo": + raise ValueError("Verl template must declare framework: verl and stage: grpo") + + trainer = state.get("trainer") or {} + run_dir = Path(str(trainer.get("trainer_output_dir") or trainer.get("output_dir") or "./outputs")).resolve() + version_id = str(trainer.get("trainer_version_id") or run_dir.name) + train_value = trainer.get("train_input_dataset_path") + train_path = Path(str(train_value)).expanduser() if train_value else None + eval_value = trainer.get("train_input_eval_dataset_path") + eval_path = Path(str(eval_value)).expanduser() if eval_value else None + # Verl's copy_to_local helper rejects model sources ending in "/". Normalize + # the user-provided value here so every generated/approved config is safe, + # including values restored from an older task snapshot. + model_path = str(trainer.get("train_input_model_name") or "").strip().rstrip("/") + + config["entrypoint"] = str(trainer.get("verl_entrypoint") or config.get("entrypoint") or "verl.trainer.main_ppo") + environment = config.setdefault("environment", {}) + environment["verl_dir"] = str(trainer.get("verl_dir") or "") + environment["verl_env_path"] = str(trainer.get("verl_env_path") or "verl") + environment["cuda_visible_devices"] = str(trainer.get("CUDA_VISIBLE_DEVICES") or "0") + environment["model_backend"] = str(trainer.get("verl_model_backend") or "fsdp") + environment["metrics_jsonl"] = str(run_dir / "metrics" / "verl_metrics.jsonl") + + overrides = config.setdefault("overrides", {}) + overrides["algorithm.adv_estimator"] = "grpo" + overrides["data.train_files"] = [str(train_path.resolve())] if train_path else [] + overrides["data.val_files"] = [str(eval_path.resolve())] if eval_path else [] + overrides["actor_rollout_ref.model.path"] = model_path + overrides["actor_rollout_ref.rollout.name"] = str(trainer.get("verl_rollout_backend") or "vllm") + overrides["trainer.experiment_name"] = version_id + overrides["trainer.default_local_dir"] = str(run_dir / "checkpoints") + overrides["trainer.n_gpus_per_node"] = _gpu_count(trainer.get("CUDA_VISIBLE_DEVICES")) + overrides["trainer.max_actor_ckpt_to_keep"] = int(trainer.get("verl_max_actor_ckpt_to_keep") or 10) + + reward_path = trainer.get("verl_reward_function_path") + raw_reward_mode = str(trainer.get("verl_reward_mode") or "").strip().lower() + reward_mode = raw_reward_mode or ("custom" if reward_path else "auto") + if reward_mode not in {"auto", "preset", "custom"}: + raise ValueError("verl_reward_mode must be auto, preset, or custom") + reward_kwargs = trainer.get("verl_reward_kwargs") or {} + if not isinstance(reward_kwargs, dict): + raise ValueError("verl_reward_kwargs must be a mapping") + reward_kwargs = copy.deepcopy(reward_kwargs) + + if reward_mode == "custom": + if not reward_path: + raise ValueError("custom reward mode requires verl_reward_function_path") + resolved_reward_path = str(Path(str(reward_path)).expanduser().resolve()) + reward_function_name = str(trainer.get("verl_reward_function_name") or "compute_score") + reward_preset = "" + else: + reward_preset = normalize_reward_preset( + "auto" if reward_mode == "auto" else trainer.get("verl_reward_preset") + ) + resolved_reward_path = str(_REWARD_ROUTER_PATH.resolve()) + reward_function_name = "compute_score" + reward_kwargs["preset"] = reward_preset + + overrides["reward.custom_reward_function.path"] = resolved_reward_path + overrides["reward.custom_reward_function.name"] = reward_function_name + # The current Verl structured config does not declare reward_kwargs even + # though get_custom_reward_fn supports it, so Hydra requires the + prefix. + overrides["+reward.custom_reward_function.reward_kwargs"] = reward_kwargs + + loopai_reward = config.setdefault("loopai_reward", {}) + loopai_reward["mode"] = reward_mode + loopai_reward["preset"] = reward_preset or None + loopai_reward["function_path"] = resolved_reward_path + loopai_reward["function_name"] = reward_function_name + + result = config.setdefault("result", {}) + result["selection_metric"] = str( + trainer.get("verl_selection_metric") or "val-core/*/acc/mean@*" + ) + result["selection_mode"] = str(trainer.get("verl_selection_mode") or "max") + config = strip_retired_tracking_config(config) + config.setdefault("overrides", {})["trainer.logger"] = ["console", "file"] + assert_no_retired_tracking(config) + return config + + +def save_verl_grpo_config(config: Dict[str, Any], output_path: str) -> str: + path = Path(output_path).expanduser().resolve() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + yaml.safe_dump(config, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + return str(path) + + +def generate_verl_config_explanation(config: Dict[str, Any]) -> str: + overrides = config.get("overrides") or {} + result = config.get("result") or {} + loopai_reward = config.get("loopai_reward") or {} + lines = [ + "Verl GRPO configuration", + f"- entrypoint: {config.get('entrypoint')}", + f"- train files: {overrides.get('data.train_files')}", + f"- validation files: {overrides.get('data.val_files')}", + f"- model: {overrides.get('actor_rollout_ref.model.path')}", + f"- rollout backend: {overrides.get('actor_rollout_ref.rollout.name')}", + f"- GPUs per node: {overrides.get('trainer.n_gpus_per_node')}", + f"- checkpoints: {overrides.get('trainer.default_local_dir')}", + f"- max actor checkpoints: {overrides.get('trainer.max_actor_ckpt_to_keep')}", + f"- reward mode: {loopai_reward.get('mode')}", + f"- reward preset: {loopai_reward.get('preset') or 'custom'}", + f"- reward function: {overrides.get('reward.custom_reward_function.path')}", + f"- selection metric: {result.get('selection_metric')} ({result.get('selection_mode')})", + ] + return "\n".join(lines) + "\n" diff --git a/loopai/skills/Trainer/utils/verl_data_checker.py b/loopai/skills/Trainer/utils/verl_data_checker.py new file mode 100644 index 00000000..0b1a6974 --- /dev/null +++ b/loopai/skills/Trainer/utils/verl_data_checker.py @@ -0,0 +1,434 @@ +from __future__ import annotations + +import ast +import json +import math +import os +import subprocess +from pathlib import Path +from typing import Any, Dict + +from loopai.common.tracking import strip_retired_tracking_environment +from loopai.skills.Trainer.rewards import ( + is_verl_builtin_data_source, + normalize_reward_preset, +) + + +_REQUIRED_COLUMNS = {"prompt", "data_source", "reward_model"} +_REWARD_ROUTER_PATH = Path(__file__).resolve().parents[1] / "rewards" / "router.py" +_SEMANTIC_SAMPLE_LIMIT = 100 +_REWARD_SMOKE_SCRIPT = r""" +import importlib.util +import json +import math +import sys + +payload = json.load(sys.stdin) +sys.path.insert(0, payload["verl_dir"]) +spec = importlib.util.spec_from_file_location("loopai_reward_smoke", payload["path"]) +if spec is None or spec.loader is None: + raise RuntimeError("unable to load reward module") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +function = getattr(module, payload["function_name"]) +result = function( + data_source=payload["data_source"], + solution_str=payload["solution_str"], + ground_truth=payload["ground_truth"], + extra_info=payload.get("extra_info") or {}, + **(payload.get("reward_kwargs") or {}), +) +if isinstance(result, dict): + result = result["score"] +elif isinstance(result, (list, tuple)): + result = result[0] +score = float(result) +if not math.isfinite(score): + raise ValueError("reward score is not finite") +print(json.dumps({"score": score})) +""" + + +def _is_missing(value: Any) -> bool: + if value is None: + return True + if isinstance(value, str): + return not value.strip() + if isinstance(value, (list, tuple, dict, set)): + return not value + return False + + +def _validate_sample(row: Dict[str, Any], label: str, index: int) -> list[str]: + errors: list[str] = [] + prompt = row.get("prompt") + if not isinstance(prompt, list) or not prompt: + errors.append(f"{label} row {index} prompt must be a non-empty chat message list") + else: + for message_index, message in enumerate(prompt): + if not isinstance(message, dict): + errors.append( + f"{label} row {index} prompt[{message_index}] must be a mapping" + ) + break + if _is_missing(message.get("role")) or _is_missing(message.get("content")): + errors.append( + f"{label} row {index} prompt[{message_index}] requires non-empty role/content" + ) + break + + if _is_missing(row.get("data_source")): + errors.append(f"{label} row {index} data_source must be non-empty") + + reward_model = row.get("reward_model") + if not isinstance(reward_model, dict): + errors.append(f"{label} row {index} reward_model must be a mapping") + elif _is_missing(reward_model.get("ground_truth")): + errors.append(f"{label} row {index} reward_model.ground_truth must be non-empty") + return errors + + +def _first_reward_sample(path_value: str) -> Dict[str, Any] | None: + try: + import pyarrow.parquet as parquet + + parquet_file = parquet.ParquetFile(Path(path_value).expanduser().resolve()) + columns = ["data_source", "reward_model"] + if "extra_info" in parquet_file.schema_arrow.names: + columns.append("extra_info") + for batch in parquet_file.iter_batches(batch_size=1, columns=columns): + rows = batch.to_pylist() + return rows[0] if rows else None + except Exception: + return None + return None + + +def _smoke_solution(data_source: str, ground_truth: Any, preset: str) -> str: + value = ground_truth + if isinstance(value, dict) and "target" in value: + value = value["target"] + if isinstance(value, (list, tuple)): + value = value[0] if value else "" + answer = str(value) + if preset == "gsm8k_exact" or data_source == "openai/gsm8k": + return f"#### {answer}" + if preset == "qa_exact_match" or data_source.startswith("searchR1_"): + return f"{answer}" + return f"\\boxed{{{answer}}}" + + +def _smoke_test_preset( + train_path: str, + reward: Dict[str, Any], + reward_kwargs: Dict[str, Any] | None, + verl_dir: str | None, + verl_env_path: str | None, +) -> Dict[str, Any]: + if not verl_dir or not verl_env_path: + return { + "status": "skipped", + "reason": "verl_dir and verl_env_path are required for preset smoke testing", + } + sample = _first_reward_sample(train_path) + if not sample: + return {"status": "skipped", "reason": "no readable reward sample"} + reward_model = sample.get("reward_model") or {} + ground_truth = reward_model.get("ground_truth") + data_source = str(sample.get("data_source") or "") + kwargs = dict(reward_kwargs or {}) + kwargs["preset"] = reward.get("preset") or "auto" + payload = { + "verl_dir": str(Path(verl_dir).expanduser().resolve()), + "path": reward["path"], + "function_name": reward["function_name"], + "data_source": data_source, + "solution_str": _smoke_solution(data_source, ground_truth, kwargs["preset"]), + "ground_truth": ground_truth, + "extra_info": sample.get("extra_info") or {}, + "reward_kwargs": kwargs, + } + try: + from loopai.skills.Trainer.utils.verl_launcher import _resolve_python_command + + python_command, env_root = _resolve_python_command(verl_env_path) + environment = strip_retired_tracking_environment(os.environ) + environment["PYTHONPATH"] = os.pathsep.join( + item + for item in (payload["verl_dir"], environment.get("PYTHONPATH", "")) + if item + ) + if env_root: + environment["PATH"] = os.pathsep.join( + item + for item in (str(Path(env_root) / "bin"), environment.get("PATH", "")) + if item + ) + completed = subprocess.run( + [*python_command, "-c", _REWARD_SMOKE_SCRIPT], + input=json.dumps(payload, ensure_ascii=False), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=payload["verl_dir"], + env=environment, + timeout=60, + check=False, + ) + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout or "unknown error").strip() + return {"status": "failed", "error": detail[-2000:]} + output = json.loads((completed.stdout or "{}").strip().splitlines()[-1]) + score = float(output["score"]) + if not math.isfinite(score): + raise ValueError("reward score is not finite") + return {"status": "passed", "score": score, "data_source": data_source} + except Exception as exc: + return {"status": "failed", "error": str(exc)} + + +def _check_parquet(path_value: str, label: str) -> Dict[str, Any]: + path = Path(path_value).expanduser().resolve() + result: Dict[str, Any] = { + "label": label, + "path": str(path), + "is_valid": True, + "columns": [], + "rows": None, + "data_sources": [], + "semantic_sample_size": 0, + "errors": [], + "warnings": [], + } + if not path.is_file(): + result["errors"].append(f"{label} dataset does not exist: {path}") + elif path.suffix.lower() != ".parquet": + result["errors"].append(f"{label} dataset must be a .parquet file: {path}") + else: + try: + with path.open("rb") as stream: + header = stream.read(4) + stream.seek(-4, 2) + footer = stream.read(4) + if header != b"PAR1" or footer != b"PAR1": + result["errors"].append(f"{label} file does not have valid Parquet magic bytes: {path}") + except (OSError, ValueError) as exc: + result["errors"].append(f"Unable to read {label} dataset: {exc}") + + if not result["errors"]: + try: + import pyarrow.parquet as parquet + + parquet_file = parquet.ParquetFile(path) + columns = set(parquet_file.schema_arrow.names) + result["columns"] = sorted(columns) + result["rows"] = parquet_file.metadata.num_rows + if result["rows"] == 0: + result["errors"].append(f"{label} Parquet dataset is empty: {path}") + missing = sorted(_REQUIRED_COLUMNS - columns) + if missing: + result["errors"].append( + f"{label} Parquet is missing required Verl columns: {', '.join(missing)}" + ) + else: + data_sources = set() + for batch in parquet_file.iter_batches( + batch_size=4096, + columns=["data_source"], + ): + for row in batch.to_pylist(): + value = row.get("data_source") + if not _is_missing(value): + data_sources.add(str(value).strip()) + result["data_sources"] = sorted(data_sources) + + sampled = [] + for batch in parquet_file.iter_batches( + batch_size=_SEMANTIC_SAMPLE_LIMIT, + columns=sorted(_REQUIRED_COLUMNS), + ): + sampled = batch.to_pylist()[:_SEMANTIC_SAMPLE_LIMIT] + break + result["semantic_sample_size"] = len(sampled) + semantic_errors: list[str] = [] + for index, row in enumerate(sampled): + semantic_errors.extend(_validate_sample(row, label, index)) + if len(semantic_errors) >= 10: + result["warnings"].append( + f"{label} semantic validation stopped after 10 errors" + ) + break + result["errors"].extend(semantic_errors[:10]) + except ImportError: + result["warnings"].append( + "pyarrow is not installed in the LoopAI environment; schema columns will be " + "validated by Verl when training starts." + ) + except Exception as exc: + result["errors"].append(f"Unable to inspect {label} Parquet schema: {exc}") + + result["is_valid"] = not result["errors"] + return result + + +def _check_reward( + path_value: str | None, + function_name: str, + reward_mode: str | None = None, + reward_preset: str | None = None, + reward_kwargs: Dict[str, Any] | None = None, +) -> Dict[str, Any]: + mode = str(reward_mode or ("custom" if path_value else "auto")).strip().lower() + result: Dict[str, Any] = { + "path": None, + "function_name": function_name, + "mode": mode, + "preset": None, + "is_valid": True, + "errors": [], + "warnings": [], + } + if mode not in {"auto", "preset", "custom"}: + result["errors"].append("reward mode must be auto, preset, or custom") + result["is_valid"] = False + return result + if reward_kwargs is not None and not isinstance(reward_kwargs, dict): + result["errors"].append("reward kwargs must be a mapping") + + if mode == "custom": + if not path_value: + result["errors"].append("custom reward mode requires a Python file path") + result["is_valid"] = False + return result + resolved_path = path_value + else: + try: + preset = normalize_reward_preset("auto" if mode == "auto" else reward_preset) + result["preset"] = preset + except ValueError as exc: + result["errors"].append(str(exc)) + result["is_valid"] = False + return result + resolved_path = str(_REWARD_ROUTER_PATH) + function_name = "compute_score" + result["function_name"] = function_name + + path = Path(resolved_path).expanduser().resolve() + result["path"] = str(path) + if not path.is_file(): + result["errors"].append(f"Custom reward Python file does not exist: {path}") + elif path.suffix.lower() != ".py": + result["errors"].append(f"Custom reward must be a Python file: {path}") + else: + try: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + defined = { + node.name + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + if function_name not in defined: + result["errors"].append( + f"Reward function '{function_name}' was not found in {path}" + ) + except (OSError, SyntaxError) as exc: + result["errors"].append(f"Unable to validate reward function: {exc}") + result["is_valid"] = not result["errors"] + return result + + +def check_verl_grpo_inputs( + train_path: str, + eval_path: str, + reward_path: str | None = None, + reward_function_name: str = "compute_score", + reward_mode: str | None = None, + reward_preset: str | None = None, + reward_kwargs: Dict[str, Any] | None = None, + verl_dir: str | None = None, + verl_env_path: str | None = None, +) -> Dict[str, Any]: + """Validate the files required by the initial LoopAI Verl GRPO adapter.""" + train = _check_parquet(train_path, "train") + validation = _check_parquet(eval_path, "validation") + reward = _check_reward( + reward_path, + reward_function_name, + reward_mode, + reward_preset, + reward_kwargs, + ) + if reward.get("mode") in {"auto", "preset"} and reward.get("preset") in { + "auto", + "verl_builtin", + }: + data_sources = set(train.get("data_sources") or []) | set( + validation.get("data_sources") or [] + ) + unknown = sorted( + source for source in data_sources if not is_verl_builtin_data_source(source) + ) + if unknown: + reward["errors"].append( + "Verl's built-in reward router does not support data_source values: " + + ", ".join(unknown) + + "; select a named preset or custom reward" + ) + reward["is_valid"] = False + if reward.get("mode") in {"auto", "preset"} and reward.get("is_valid"): + reward["smoke_test"] = _smoke_test_preset( + train_path, + reward, + reward_kwargs, + verl_dir, + verl_env_path, + ) + if reward["smoke_test"].get("status") == "failed": + reward["errors"].append( + "Reward preset smoke test failed: " + + str(reward["smoke_test"].get("error") or "unknown error") + ) + reward["is_valid"] = False + errors = train["errors"] + validation["errors"] + reward["errors"] + warnings = train["warnings"] + validation["warnings"] + reward["warnings"] + return { + "framework": "verl", + "stage": "grpo", + "is_valid": not errors, + "train": train, + "validation": validation, + "reward": reward, + "errors": errors, + "warnings": warnings, + "total_samples": train.get("rows"), + } + + +def generate_verl_data_report(result: Dict[str, Any]) -> str: + status = "PASS" if result.get("is_valid") else "FAIL" + lines = [f"Verl GRPO input check: {status}"] + for key in ("train", "validation"): + item = result.get(key) or {} + lines.append(f"- {key}: {item.get('path')}") + if item.get("rows") is not None: + lines.append(f" rows: {item.get('rows')}") + if item.get("columns"): + lines.append(f" columns: {', '.join(item['columns'])}") + if item.get("data_sources"): + lines.append(f" data_sources: {', '.join(item['data_sources'])}") + if item.get("semantic_sample_size") is not None: + lines.append(f" semantic rows checked: {item.get('semantic_sample_size')}") + reward = result.get("reward") or {} + lines.append( + f"- reward: mode={reward.get('mode')}, preset={reward.get('preset') or 'custom'}, " + f"path={reward.get('path')} ({reward.get('function_name')})" + ) + smoke_test = reward.get("smoke_test") or {} + if smoke_test: + lines.append(f" smoke test: {smoke_test.get('status')}") + for error in result.get("errors") or []: + lines.append(f"ERROR: {error}") + for warning in result.get("warnings") or []: + lines.append(f"WARNING: {warning}") + return "\n".join(lines) + "\n" diff --git a/loopai/skills/Trainer/utils/verl_exporter.py b/loopai/skills/Trainer/utils/verl_exporter.py new file mode 100644 index 00000000..ee0731d7 --- /dev/null +++ b/loopai/skills/Trainer/utils/verl_exporter.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any, Dict + +from .verl_launcher import build_verl_launch + + +def _has_model_weights(path: Path) -> bool: + return any( + (path / name).is_file() + for name in ( + "model.safetensors", + "model.safetensors.index.json", + "pytorch_model.bin", + "pytorch_model.bin.index.json", + ) + ) + + +def export_verl_checkpoint( + config_path: str, + checkpoint: Dict[str, Any], + log_path: str, + app_config: Dict[str, Any] | None = None, +) -> str: + """Merge the selected Verl FSDP actor into one Hugging Face model directory.""" + checkpoint_root = Path(str(checkpoint.get("checkpoint_path") or "")).expanduser().resolve() + actor_dir = checkpoint_root / "actor" + if not actor_dir.is_dir(): + raise FileNotFoundError(f"Selected Verl actor checkpoint does not exist: {actor_dir}") + + target_dir = checkpoint_root / "merged_huggingface" + if target_dir.is_dir() and _has_model_weights(target_dir): + return str(target_dir) + + training_cmd, cwd, env = build_verl_launch(config_path, app_config) + try: + module_index = training_cmd.index("-m") + except ValueError as exc: + raise RuntimeError("Unable to derive Verl Python command from launch config") from exc + cmd = [ + *training_cmd[:module_index], + "-m", + "verl.model_merger", + "merge", + "--backend", + "fsdp", + "--local_dir", + str(actor_dir), + "--target_dir", + str(target_dir), + ] + output_log = Path(log_path).expanduser().resolve() + output_log.parent.mkdir(parents=True, exist_ok=True) + with output_log.open("w", encoding="utf-8") as stream: + completed = subprocess.run( + cmd, + cwd=cwd, + env=env, + stdout=stream, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"Verl checkpoint merge exited with code {completed.returncode}; see {output_log}" + ) + if not _has_model_weights(target_dir): + raise RuntimeError(f"Verl merger completed but no Hugging Face weights were found in {target_dir}") + return str(target_dir) diff --git a/loopai/skills/Trainer/utils/verl_launcher.py b/loopai/skills/Trainer/utils/verl_launcher.py new file mode 100644 index 00000000..d41f80fa --- /dev/null +++ b/loopai/skills/Trainer/utils/verl_launcher.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +import json +import os +import re +import shutil +import sys +from pathlib import Path +from typing import Any, Dict, Tuple + +import yaml + +from loopai.common.tracking import assert_no_retired_tracking, strip_retired_tracking_environment + + +_ALLOWED_ENTRYPOINTS = { + "verl.trainer.main_ppo", + "verl.trainer.main_ppo_sync", +} +_OVERRIDE_KEY = re.compile(r"^\+?[A-Za-z0-9_.-]+$") +_DICT_KEY = re.compile(r"^[A-Za-z0-9_.-]+$") + + +def load_verl_launch_config(config_path: str) -> Dict[str, Any]: + """Load and validate LoopAI's user-approved Verl launch specification.""" + path = Path(config_path).expanduser().resolve() + if path.suffix.lower() not in {".yaml", ".yml"}: + raise ValueError(f"Verl launch config must be YAML: {path}") + if not path.is_file(): + raise FileNotFoundError(f"Verl launch config does not exist: {path}") + + config = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(config, dict): + raise ValueError(f"Verl launch config must contain a YAML mapping: {path}") + assert_no_retired_tracking(config) + if config.get("framework") != "verl" or config.get("stage") != "grpo": + raise ValueError("Verl launch config must declare framework: verl and stage: grpo") + + entrypoint = str(config.get("entrypoint") or "") + if entrypoint not in _ALLOWED_ENTRYPOINTS: + raise ValueError( + f"Unsupported Verl entrypoint: {entrypoint}; " + f"allowed values: {', '.join(sorted(_ALLOWED_ENTRYPOINTS))}" + ) + overrides = config.get("overrides") + if not isinstance(overrides, dict) or not overrides: + raise ValueError("Verl launch config must contain non-empty overrides") + invalid_keys = [str(key) for key in overrides if not _OVERRIDE_KEY.fullmatch(str(key))] + if invalid_keys: + raise ValueError(f"Invalid Hydra override keys: {', '.join(invalid_keys)}") + result_config = config.get("result") + if isinstance(result_config, dict) and result_config.get("selection_mode") is not None: + selection_mode = str(result_config["selection_mode"]).strip().lower() + if selection_mode not in {"max", "min"}: + raise ValueError( + "Verl result.selection_mode must be max or min, " + f"got: {result_config['selection_mode']}" + ) + result_config["selection_mode"] = selection_mode + return config + + +def _hydra_value(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, list): + return "[" + ",".join(_hydra_value(item) for item in value) + "]" + if isinstance(value, dict): + invalid_keys = [str(key) for key in value if not _DICT_KEY.fullmatch(str(key))] + if invalid_keys: + raise ValueError( + "Hydra mapping keys may contain only letters, numbers, dots, underscores, and hyphens: " + + ", ".join(invalid_keys) + ) + return "{" + ",".join( + f"{key}:{_hydra_value(item)}" for key, item in value.items() + ) + "}" + return json.dumps(str(value), ensure_ascii=False) + + +def _resolve_python_command(env_path: str | None) -> Tuple[list[str], str | None]: + if not env_path: + return [sys.executable], None + + raw = str(env_path).strip() + requested = Path(raw).expanduser() + if requested.is_absolute() or "/" in raw: + requested = requested.resolve() + bin_dir = requested if requested.name == "bin" else requested / "bin" + env_root = bin_dir.parent + for name in ("python", "python3"): + candidate = bin_dir / name + if candidate.is_file(): + return [str(candidate)], str(env_root) + raise FileNotFoundError(f"No python executable found under Verl environment: {bin_dir}") + + conda_from_path = shutil.which("conda") + conda_candidates = ( + os.getenv("CONDA_EXE"), + str(Path.home() / "miniconda3" / "bin" / "conda"), + str(Path.home() / "anaconda3" / "bin" / "conda"), + ) + conda = conda_from_path or next( + (candidate for candidate in conda_candidates if candidate and Path(candidate).is_file()), + None, + ) + if not conda: + raise FileNotFoundError( + f"Conda executable was not found while resolving Verl environment '{raw}'" + ) + return [conda, "run", "--no-capture-output", "-n", raw, "python"], None + + +def build_verl_launch( + config_path: str, + app_config: Dict[str, Any] | None = None, +) -> tuple[list[str], str, dict[str, str]]: + """Translate an approved YAML spec into a shell-free Verl Hydra command.""" + config = load_verl_launch_config(config_path) + app_config = app_config or {} + environment = config.get("environment") or {} + + verl_dir = Path(str( + environment.get("verl_dir") or app_config.get("verl_dir") or "" + )).expanduser().resolve() + if not (verl_dir / "verl" / "trainer").is_dir(): + raise FileNotFoundError(f"Invalid Verl repository directory: {verl_dir}") + + python_command, env_root = _resolve_python_command( + str(environment.get("verl_env_path") or app_config.get("verl_env_path") or "") or None + ) + cmd = [*python_command, "-m", str(config["entrypoint"])] + cmd.extend( + f"{key}={_hydra_value(value)}" + for key, value in (config.get("overrides") or {}).items() + ) + + env = strip_retired_tracking_environment(os.environ) + cuda_devices = environment.get("cuda_visible_devices") or app_config.get("CUDA_VISIBLE_DEVICES") + if cuda_devices is not None and str(cuda_devices).strip(): + env["CUDA_VISIBLE_DEVICES"] = str(cuda_devices) + env["PYTHONNOUSERSITE"] = "True" + env["PYTHONPATH"] = os.pathsep.join( + item for item in (str(verl_dir), env.get("PYTHONPATH", "")) if item + ) + if env_root: + env["PATH"] = os.pathsep.join( + item for item in (str(Path(env_root) / "bin"), env.get("PATH", "")) if item + ) + metrics_jsonl = str(environment.get("metrics_jsonl") or "").strip() + if metrics_jsonl: + metrics_path = Path(metrics_jsonl).expanduser().resolve() + metrics_path.parent.mkdir(parents=True, exist_ok=True) + env["VERL_FILE_LOGGER_PATH"] = str(metrics_path) + + return cmd, str(verl_dir), env diff --git a/loopai/skills/Trainer/worker_entry.py b/loopai/skills/Trainer/worker_entry.py new file mode 100644 index 00000000..377674b7 --- /dev/null +++ b/loopai/skills/Trainer/worker_entry.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import argparse +import os +import traceback +from pathlib import Path +from typing import Any, Dict + +from loopai.common.exception import ErrorCode, build_error_payload, build_success_payload +from loopai.common.tracking import strip_retired_tracking_fields +from loopai.skills.Trainer.utils.persistent_worker import ( + load_worker_request, + update_run_state, + utc_now_iso, + worker_paths, + write_worker_result, +) + + +def _final_event_data(state: Dict[str, Any]) -> Dict[str, Any]: + trainer = state.setdefault("trainer", {}) + return { + "task_id": state.get("task_id"), + "trainer_version_id": trainer.get("trainer_version_id"), + "trainer_training_success": trainer.get("trainer_training_success"), + "trainer_training_final_status": trainer.get("trainer_training_final_status"), + "trainer_result_summary": trainer.get("trainer_result_summary"), + "trainer_best_checkpoint": trainer.get("trainer_best_checkpoint"), + "trainer_model_export_error": trainer.get("trainer_model_export_error"), + } + + +def run_worker(request_path: str | Path) -> int: + request = Path(request_path).expanduser().resolve() + payload = load_worker_request(request) + state = payload.get("state") + if not isinstance(state, dict): + raise ValueError("Trainer worker request is missing state") + state = strip_retired_tracking_fields(state) + + trainer = state.setdefault("trainer", {}) + run_dir = trainer.get("trainer_output_dir") or trainer.get("output_dir") + if not run_dir: + raise ValueError("Trainer worker request is missing trainer_output_dir") + paths = worker_paths(run_dir) + + # The request is transient; remove it as soon as the worker has loaded it. + try: + request.unlink() + except OSError: + pass + + os.environ["LOOPAI_TRAINER_WORKER_CHILD"] = "1" + trainer["trainer_worker_pid"] = os.getpid() + trainer["trainer_run_state_path"] = str(paths["state"]) + trainer["trainer_worker_log_path"] = str(paths["log"]) + update_run_state( + paths["state"], + { + "status": "running", + "worker_pid": os.getpid(), + "started_at": utc_now_iso(), + }, + ) + + try: + from loopai.skills.Trainer.nodes.training_execution_node import _training_execution_inline + from loopai.skills.Trainer.runner import ( + _update_trainer_task_state, + finalize_trainer_result_state, + ) + from loopai.skills.Trainer.utils.stream_events import emit_trainer_event + + result = _training_execution_inline(state, writer=lambda _payload: None) + result.setdefault("trainer", {}).pop("_trainer_approved_config_yaml", None) + result.setdefault("trainer", {}).pop("_trainer_approved_config_sha256", None) + finalize_trainer_result_state(result, task_id=result.get("task_id")) + + result_trainer = result.setdefault("trainer", {}) + success = bool(result_trainer.get("trainer_training_success")) + final_status = result_trainer.get("trainer_training_final_status") or {} + reported_status = str(final_status.get("status") or "").lower() + status = "completed" if success else ( + "cancelled" if reported_status == "cancelled" else "failed" + ) + final_data = _final_event_data(result) + if success: + trainer_payload = build_success_payload( + data=final_data, + message="Trainer skill completed.", + ) + else: + existing_payload = result_trainer.get("trainer_result") + if isinstance(existing_payload, dict) and existing_payload.get("ok") is False: + trainer_payload = existing_payload + else: + error_detail = ( + result_trainer.get("train_output_training_error") + or final_status.get("error_message") + or f"training finished with status: {status}" + ) + trainer_payload = build_error_payload( + RuntimeError(str(error_detail)), + code=( + ErrorCode.INTERRUPTED + if status == "cancelled" + else ErrorCode.UNHANDLED_EXCEPTION + ), + recoverable=status != "cancelled", + message="Trainer skill failed.", + ) + result_trainer["trainer_result"] = trainer_payload + result_trainer["trainer_last_error"] = trainer_payload.get("error") or {} + + runtime = result_trainer.pop("_trainer_worker_runtime", None) + if isinstance(runtime, dict): + try: + _update_trainer_task_state(runtime, result_trainer) + except Exception as state_exc: + # Training artifacts and pkl events remain authoritative even + # when the optional database state update is temporarily down. + result_trainer["trainer_state_update_error"] = str(state_exc) + + emit_trainer_event( + lambda _payload: None, + result, + "trainer.run", + status, + progress=1.0, + message=( + "Trainer worker completed." + if success + else "Trainer worker was cancelled." + if status == "cancelled" + else "Trainer worker finished with failure." + ), + data=final_data, + error=trainer_payload.get("error"), + ) + + write_worker_result( + paths["result"], + {"ok": True, "state": strip_retired_tracking_fields(result)}, + ) + update_run_state( + paths["state"], + { + "status": status, + "finished_at": utc_now_iso(), + "result_path": str(paths["result"]), + }, + ) + return 0 + except BaseException as exc: + error = { + "type": type(exc).__name__, + "detail": str(exc), + "traceback": "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)), + } + print(error["traceback"], flush=True) + try: + write_worker_result(paths["result"], {"ok": False, "error": error}) + update_run_state( + paths["state"], + { + "status": "failed", + "finished_at": utc_now_iso(), + "error": error, + "result_path": str(paths["result"]), + }, + ) + except Exception: + traceback.print_exc() + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run one persistent LoopAI Trainer task") + parser.add_argument("--request", required=True, help="Path to the trusted worker request pickle") + args = parser.parse_args() + return run_worker(args.request) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/Trainer/SKILL.md b/skills/Trainer/SKILL.md index 5275a802..e1800429 100644 --- a/skills/Trainer/SKILL.md +++ b/skills/Trainer/SKILL.md @@ -1,6 +1,6 @@ --- name: trainer -description: Use this skill when the user wants LoopAI to validate training data, generate and approve LLaMA-Factory training YAML, start SFT/training, monitor Trainer progress events, or inspect Trainer failures from starter.yaml or runtime state. +description: Use this skill when the user wants LoopAI to validate training data, generate and approve training YAML, run LLaMA-Factory SFT or Verl GRPO, configure rewards, monitor or reconnect to Trainer workers, compare checkpoints, export a trained model, or inspect Trainer failures from starter.yaml or runtime state. --- # Trainer Skill @@ -11,11 +11,12 @@ Trainer Skill is the Codex-facing entry point and the canonical Python implement Use this skill for: -- Starting SFT or Trainer training from `starter.yaml` -- Validating Trainer configuration and dataset paths -- Generating LLaMA-Factory training YAML -- Running Trainer and reading structured StreamEvent progress -- Returning structured Trainer errors +- Running `sft + llamafactory` or `grpo + verl`; do not mix backend/stage pairs +- Validating SFT JSON/JSONL or GRPO Parquet data and reward configuration +- Generating and approving backend-specific training YAML +- Running or reconnecting to the persistent Trainer worker +- Reading local metrics, selecting checkpoints, and exporting Verl FSDP actors +- Returning structured Trainer errors without relying on SwanLab or Trainer MCP Do not use this skill for Judger, Analyzer, data crawling, or broad project refactors. @@ -26,11 +27,13 @@ loopai/skills/Trainer/ ├── __init__.py # prepare() / run_prepared() / run() / result helpers ├── trainer_agent.py # LangGraph subgraph used by Starter and the skill runner ├── nodes/ # data validation, config generation, training execution -├── utils/ # events, task manager, parsers, training utilities +├── rewards/ # stable LoopAI routing to Verl reward implementations +├── utils/ # persistent worker, Verl/SFT launchers, events, parsers ├── templates/ # bundled training templates ├── results.py # parses metrics and selects the best checkpoint ├── runner.py # skill entry that runs the Trainer subgraph -└── runtime_config.py # resolves kwargs/env/state/starter.yaml +├── runtime_config.py # resolves kwargs/env/state/starter.yaml +└── worker_entry.py # independent process that owns training and finalization ``` The root skill description lives at: @@ -47,7 +50,9 @@ For every user-initiated training round, use this two-stage workflow: 2. Read `result["trainer"]["trainer_result"]["data"]` and show the user: - `config_path` - the complete `config_yaml` in a YAML code block - - the important values such as dataset/model paths, learning rate, epochs, batch size, LoRA fields, devices, output directory, and `save_total_limit` + - the selected `train_framework` and `train_stage` + - for SFT: dataset/model paths, learning rate, epochs, batch size, LoRA fields, devices, output directory, and `save_total_limit` + - for GRPO: train/validation Parquet paths, model, reward mode/preset or custom function, rollout backend, GPUs, batch/token limits, save/test frequency, checkpoint directory, selection metric, and checkpoint retention 3. Stop and wait for explicit user approval. Do not treat a previous round's approval as approval for a new round. 4. If the user requests edits, update the generated YAML, call `inspect_prepared_config()`, show the complete updated YAML, and wait for approval again. 5. Only after approval, call `run_prepared()` with the displayed `config_path`, its displayed `config_sha256`, and the `trainer_version_id` returned by `prepare()`. @@ -55,7 +60,11 @@ For every user-initiated training round, use this two-stage workflow: Never call `run()` directly for an interactive, user-initiated training request. Keep `run()` only as a backward-compatible entry point for explicitly non-interactive callers that intentionally opt out of human approval. -The SHA-256 check is part of the approval boundary. If the YAML changes after it is shown, `run_prepared()` must reject it; show the changed YAML and request approval again. +The SHA-256 check is part of the approval boundary. If the YAML changes before +`run_prepared()` validates it, reject it, show the changed YAML, and request +approval again. After validation, Trainer carries the exact approved YAML text +inside the trusted Worker request and atomically materializes that snapshot for +the launcher; it must not reread a mutable source file for execution. ## Quick Start @@ -105,6 +114,50 @@ prepared = prepare( ) ``` +### Verl GRPO State + +Use both training and validation Parquet files. When `pyarrow` is available, +Trainer checks the schema, reads the distinct `data_source` values, and +semantically validates up to the first 100 rows of each file. Every sampled row +must contain a non-empty chat-message-list `prompt`, a supported `data_source`, +and `reward_model.ground_truth`. Without `pyarrow`, Trainer checks the file and +Parquet magic bytes, emits a warning, and defers schema validation to Verl. + +```python +prepared = prepare( + state={ + "task_id": "trainer_grpo_001", + "output_dir": "./outputs", + "trainer": { + "train_framework": "verl", + "train_stage": "grpo", + "verl_dir": "/path/to/verl", + "verl_env_path": "verl", + "train_input_dataset_path": "/path/to/train.parquet", + "train_input_eval_dataset_path": "/path/to/validation.parquet", + "train_input_model_name": "/path/to/model", + "train_input_task_description": "Optimize task accuracy with GRPO.", + "verl_reward_mode": "preset", + "verl_reward_preset": "math_boxed", + "CUDA_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7", + }, + }, + thread_id="trainer_grpo_001", +) +``` + +Use exactly one reward mode: + +- `auto`: route by Parquet `data_source` through Verl's built-in router. When + `pyarrow` is available, reject unsupported sources during preparation; + otherwise defer that check to Verl with a validation warning. +- `preset`: set `verl_reward_preset` to `auto`, `verl_builtin`, `gsm8k_exact`, `math_boxed`, `math_dapo`, `prime_math`, `geometry`, or `qa_exact_match`. +- `custom`: set `verl_reward_function_path` and optionally `verl_reward_function_name` (default `compute_score`) and `verl_reward_kwargs`. + +The preset router imports reward implementations lazily from the configured Verl +environment. Do not copy Verl reward source into LoopAI and do not silently fall +back from an unknown `data_source` or preset. + ## Runtime Configuration Priority: @@ -137,24 +190,35 @@ Useful environment variables: - `OUTPUT_DIR` - `TRAINER_OUTPUT_DIR` - `TRAIN_FRAMEWORK` +- `TRAIN_STAGE` - `TRAIN_DATASET_PATH` +- `TRAIN_EVAL_DATASET_PATH` - `TRAIN_MODEL_PATH` - `TRAIN_TASK_DESCRIPTION` - `TRAIN_CONFIG_TEMPLATE_PATH` - `LLAMAFACTORY_DIR` - `LLAMAFACTORY_ENV_PATH` +- `VERL_DIR` +- `VERL_ENV_PATH` or `VERL_CONDA_ENV` +- `VERL_REWARD_MODE`, `VERL_REWARD_PRESET`, and `VERL_REWARD_KWARGS` +- `VERL_REWARD_FUNCTION_PATH` and `VERL_REWARD_FUNCTION_NAME` for custom reward mode +- `TRAINER_PERSISTENT_WORKER` - `CUDA_VISIBLE_DEVICES` -- `SWANLAB_API_KEY` -- `TRAIN_USE_SWANLAB` + +Trainer metrics are driven by local files and do not require an external +experiment-tracking service. SFT reads `trainer_log.jsonl` and +`metrics/metrics.json`; Verl reads `metrics/verl_metrics.jsonl`. Required Trainer fields: - `train_framework` +- `train_stage` - `train_input_dataset_path` - `train_input_task_description` - `train_input_config_template_path` - `train_input_model_name` -- `llamafactory_dir` when `train_framework` is `llamafactory` +- for SFT: `train_framework=llamafactory`, `train_stage=sft`, and `llamafactory_dir` +- for GRPO: `train_framework=verl`, `train_stage=grpo`, `verl_dir`, `train_input_eval_dataset_path`, and a valid reward mode ## Versioned Runtime @@ -181,12 +245,25 @@ Trainer static files are written under: {output_dir}/{task_id}/trainer/{trainer_version_id}/ ``` +The task-level compatibility event pickle remains at: + +```text +{output_dir}/{task_id}/trainer.pkl +``` + +The version directory contains `run_state.json`, `worker.log`, and, after +finalization, `worker_result.pkl`. Treat pickle files as trusted local artifacts; +they are created with user-only permissions and must not be loaded from an +untrusted source. + Important state fields: - `trainer_version_id` - `trainer_output_dir` - `trainer_event_log_path` - `trainer_training_task_id` +- `trainer_run_state_path` +- `trainer_worker_log_path` Keep `context_id` as the task id when reading events or runtime state. Use `trainer_version_id` only to distinguish a specific run. @@ -209,14 +286,16 @@ Fields that usually require user or task-specific input: - `train_input_dataset_path` - `train_input_model_name` - `train_input_task_description` -- `llamafactory_dir` +- `llamafactory_dir` for SFT, or `verl_dir` and `train_input_eval_dataset_path` for GRPO Fields that Trainer can usually prefill: -- `train_framework`: defaults to `llamafactory` -- `train_input_config_template_path`: defaults to the bundled SFT YAML template +- `train_framework` / `train_stage`: default to `llamafactory` / `sft`, or infer `verl` / `grpo` from `task_type="grpo"` +- `train_input_config_template_path`: selects the bundled SFT or GRPO YAML template +- `verl_env_path`: defaults to Conda environment `verl` +- `verl_reward_mode`: defaults to `auto` +- `trainer_persistent_worker`: defaults to `true` - `CUDA_VISIBLE_DEVICES`: defaults to `0` -- `train_input_use_swanlab`: defaults to `false` If `guide["user_required_fields"]` is non-empty, ask the user or Configer to fill those fields before starting training. Do not start Trainer only with placeholder paths. @@ -278,13 +357,21 @@ The analyzer reads, when available: - `trainer_log.jsonl` - `metrics/metrics.json` -- `checkpoint-*` directories +- `metrics/verl_metrics.jsonl` +- SFT `checkpoint-*` and Verl `checkpoints/global_step_*` directories Best checkpoint selection rule: -1. Prefer the checkpoint aligned with the lowest `eval_loss`. -2. If no `eval_loss` is available, use the lowest training `loss`. -3. If no loss metric is available, choose the latest checkpoint. +1. For Verl, use `result.selection_metric` and `result.selection_mode` from the approved YAML. Fall back to the runtime `verl_selection_metric` / `verl_selection_mode` defaults (`val-core/*/acc/mean@*`, maximize) only when the YAML omits them. +2. Otherwise prefer the lowest `eval_loss`, then the lowest training `loss`, then the highest validation score or reward. +3. Align metrics and checkpoints by numeric `step`/`global_step`, not by list index. Prefer an exact step, otherwise the nearest saved checkpoint not after the best metric step, then the nearest checkpoint. +4. Ignore non-finite metric values (`NaN` / `Inf`) and reject a selection mode other than `max` or `min`. +5. If no usable metric exists, choose the latest checkpoint. + +When `result.export_huggingface=true` and the selected Verl `global_step_*` +actor is still an FSDP shard, Trainer runs `verl.model_merger` and writes a +Hugging Face model directory. Report the merged `update_model_path`; do not +pass an unmerged actor shard to Judger. When comparing multiple Trainer runs, call `analyze_results()` for each run and compare: @@ -299,10 +386,11 @@ Prefer reporting the selected checkpoint path from `trainer_best_checkpoint_path ## Execution Lifetime -Trainer runs synchronously in the foreground. A Trainer invocation is not -complete when the training process has merely started; it is complete only -after the training status becomes `completed`, `failed`, or `cancelled` and -the local Trainer runner exits. +Trainer defaults to `trainer_persistent_worker=true`. An independent local +worker owns the LLaMA-Factory/Verl process, progress persistence, result +analysis, and final state update. The normal caller still waits synchronously: +a Trainer invocation is not complete when the worker has merely started; it is +complete only after the run reaches `completed`, `failed`, or `cancelled`. - Do not launch the Trainer runner with `&`, `nohup`, or a detached shell. - If a command execution yields a running cell/session id, keep waiting on @@ -310,12 +398,23 @@ the local Trainer runner exits. - Do not finish the Codex turn while the Trainer command is still running. - Progress events with status `running` are intermediate updates, not a final tool result. - -## MCP Status (Disabled) - -The local LoopAI MCP route is currently disabled. Do not call or register -`trainer_run` / `trainer_load_events`, and do not start `loopai.mcp.server`. -Call the local Trainer skill or its script/runner entrypoints directly. +- If the caller or API session ends unexpectedly while the worker remains + alive, do not submit the same `version_id` as a new run. Reconnect to that + worker and read `run_state.json` / `worker_result.pkl`; training continues. +- If the worker itself has exited while its child training process is still + alive, Trainer refuses to launch a duplicate. Report the orphaned process and + require explicit operator recovery instead of claiming it was reattached. +- Concurrent attach/launch attempts for the same version are serialized by a + run-directory lock so only one Worker may be started. +- Use `trainer_persistent_worker=false` only for explicit local debugging of + the legacy in-process execution path. + +## Trainer MCP Status (Disabled) + +The Trainer MCP route is disabled. Do not call or register `trainer_run` / +`trainer_load_events`, and do not start an MCP server for the purpose of running +Trainer. Call the local Trainer skill or its script/runner entrypoints directly. +This restriction does not disable or remove unrelated LoopAI MCP routes. Read the task-scoped trainer section through Configer before preparing the YAML. Treat task-state confirmation and final YAML approval as separate gates: the first confirms the task inputs; the second approves the exact executable training configuration. diff --git a/tests/test_trainer_config_approval.py b/tests/test_trainer_config_approval.py index 920ee391..01f030b9 100644 --- a/tests/test_trainer_config_approval.py +++ b/tests/test_trainer_config_approval.py @@ -11,6 +11,7 @@ ) from loopai.skills.Trainer.runtime_config import _DEFAULT_TEMPLATE_PATH from loopai.skills.Trainer.trainer_agent import TrainerAgent +from loopai.skills.Trainer.nodes.training_execution_node import _materialize_training_config def test_prepare_uses_config_only_mode() -> None: @@ -46,6 +47,27 @@ def test_inspect_prepared_config_returns_exact_text_and_digest(tmp_path) -> None assert inspected["config_sha256"] == hashlib.sha256(config_yaml.encode("utf-8")).hexdigest() +def test_worker_materializes_approved_yaml_snapshot_after_source_changes(tmp_path) -> None: + source = tmp_path / "approved.yaml" + target = tmp_path / "worker" / "training.yaml" + approved_yaml = "model_name_or_path: /models/approved\nnum_train_epochs: 3\n" + source.write_text(approved_yaml, encoding="utf-8") + trainer_state = { + "_trainer_approved_config_yaml": approved_yaml, + "_trainer_approved_config_sha256": hashlib.sha256( + approved_yaml.encode("utf-8") + ).hexdigest(), + } + + source.write_text("model_name_or_path: /models/tampered\n", encoding="utf-8") + materialized = _materialize_training_config(str(source), str(target), trainer_state) + + assert materialized == str(target.resolve()) + assert target.read_text(encoding="utf-8") == approved_yaml + assert "_trainer_approved_config_yaml" not in trainer_state + assert "_trainer_approved_config_sha256" not in trainer_state + + def test_prepare_stops_after_config_generation() -> None: state = { "trainer": { @@ -96,7 +118,6 @@ def fail_if_training_starts(*args, **kwargs): "train_input_model_name": "/models/base", "train_input_task_description": "SFT a small assistant", "train_input_config_template_path": str(_DEFAULT_TEMPLATE_PATH), - "train_input_use_swanlab": False, }, }, thread_id="approval-task", @@ -145,7 +166,6 @@ def test_run_prepared_uses_exact_yaml_without_regenerating(tmp_path, monkeypatch "train_input_model_name": "/models/base", "train_input_task_description": "SFT a small assistant", "train_input_config_template_path": str(_DEFAULT_TEMPLATE_PATH), - "train_input_use_swanlab": False, }, }, thread_id=thread_id, @@ -178,3 +198,108 @@ def fake_training(state, writer=None): assert result["trainer"]["trainer_training_success"] is True assert result["trainer"]["train_output_config_path"] == approval["config_path"] + + +def test_run_prepared_preserves_structured_training_failure(tmp_path, monkeypatch) -> None: + monkeypatch.delenv("TASK_ID", raising=False) + monkeypatch.delenv("DB_PATH", raising=False) + dataset_path = tmp_path / "data.json" + dataset_path.write_text( + json.dumps([{"instruction": "hello", "input": "", "output": "world"}]), + encoding="utf-8", + ) + thread_id = f"approval-failure-{tmp_path.name}" + prepared = prepare( + state={ + "task_id": thread_id, + "output_dir": str(tmp_path), + "trainer": { + "train_framework": "llamafactory", + "llamafactory_dir": str(tmp_path), + "CUDA_VISIBLE_DEVICES": "0", + "train_input_dataset_path": str(dataset_path), + "train_input_model_name": "/models/base", + "train_input_task_description": "SFT a small assistant", + "train_input_config_template_path": str(_DEFAULT_TEMPLATE_PATH), + }, + }, + thread_id=thread_id, + ) + approval = prepared["trainer"]["trainer_result"]["data"] + + def fake_failed_training(state, writer=None): + state["trainer"]["trainer_training_success"] = False + state["trainer"]["trainer_training_final_status"] = { + "status": "failed", + "error_message": "synthetic training failure", + } + state["trainer"]["train_output_training_error"] = "synthetic training failure" + return state + + monkeypatch.setattr( + "loopai.skills.Trainer.trainer_agent.training_execution_node", + fake_failed_training, + ) + result = run_prepared( + prepared_config_path=approval["config_path"], + expected_config_sha256=approval["config_sha256"], + state=prepared, + thread_id=thread_id, + version_id=approval["trainer_version_id"], + ) + + assert result["trainer"]["trainer_training_success"] is False + assert result["trainer"]["trainer_result"]["ok"] is False + assert result["trainer"]["trainer_result"]["status"] == "failed" + assert "synthetic training failure" in result["trainer"]["trainer_last_error"]["detail"] + + +def test_run_prepared_preserves_cancelled_error_semantics(tmp_path, monkeypatch) -> None: + monkeypatch.delenv("TASK_ID", raising=False) + monkeypatch.delenv("DB_PATH", raising=False) + dataset_path = tmp_path / "data.json" + dataset_path.write_text( + json.dumps([{"instruction": "hello", "input": "", "output": "world"}]), + encoding="utf-8", + ) + thread_id = f"approval-cancelled-{tmp_path.name}" + prepared = prepare( + state={ + "task_id": thread_id, + "output_dir": str(tmp_path), + "trainer": { + "train_framework": "llamafactory", + "llamafactory_dir": str(tmp_path), + "CUDA_VISIBLE_DEVICES": "0", + "train_input_dataset_path": str(dataset_path), + "train_input_model_name": "/models/base", + "train_input_task_description": "SFT a small assistant", + "train_input_config_template_path": str(_DEFAULT_TEMPLATE_PATH), + }, + }, + thread_id=thread_id, + ) + approval = prepared["trainer"]["trainer_result"]["data"] + + def fake_cancelled_training(state, writer=None): + state["trainer"]["trainer_training_success"] = False + state["trainer"]["trainer_training_final_status"] = {"status": "cancelled"} + state["trainer"]["train_output_training_error"] = "cancelled by user" + return state + + monkeypatch.setattr( + "loopai.skills.Trainer.trainer_agent.training_execution_node", + fake_cancelled_training, + ) + result = run_prepared( + prepared_config_path=approval["config_path"], + expected_config_sha256=approval["config_sha256"], + state=prepared, + thread_id=thread_id, + version_id=approval["trainer_version_id"], + ) + + error = result["trainer"]["trainer_result"]["error"] + assert error["code"] == "INTERRUPTED" + assert error["recoverable"] is False + assert "cancelled by user" in error["detail"] diff --git a/tests/test_trainer_metrics_api.py b/tests/test_trainer_metrics_api.py new file mode 100644 index 00000000..bd5a5645 --- /dev/null +++ b/tests/test_trainer_metrics_api.py @@ -0,0 +1,84 @@ +import json + +import pytest + +from api.app.services.task.service import TaskServiceError, get_train_status + + +def _run_dir(tmp_path, task_id="task-1", version_id="version-1"): + path = tmp_path / task_id / "trainer" / version_id + (path / "metrics").mkdir(parents=True) + return path + + +def test_verl_status_prefers_structured_metrics_and_normalizes_curves(tmp_path) -> None: + run_dir = _run_dir(tmp_path) + metrics_dir = run_dir / "metrics" + records = [ + { + "step": 0, + "data": { + "val-core/math/acc/mean@1": 0.42, + }, + }, + { + "step": 1, + "data": { + "training/global_step": 1, + "critic/rewards/mean": 0.5, + "actor/pg_loss": 0.125, + }, + }, + ] + (metrics_dir / "verl_metrics.jsonl").write_text( + "\n".join(json.dumps(item) for item in records), + encoding="utf-8", + ) + (metrics_dir / "metrics.json").write_text( + json.dumps( + { + "task_info": {}, + "metrics": [ + { + "step": 999, + "pid": 123, + "log_line": "Training Progress: 1/10", + } + ], + } + ), + encoding="utf-8", + ) + + payload = get_train_status(str(tmp_path), "task-1", "version-1") + + assert payload["task_info"]["framework"] == "verl" + chart_records = [item for item in payload["metrics"] if "step" in item] + assert chart_records[0]["val_score"] == 0.42 + assert chart_records[1]["reward"] == 0.5 + assert chart_records[1]["policy_loss"] == 0.125 + assert all("pid" not in item for item in chart_records) + assert payload["metrics"][-1] == {"log_line": "Training Progress: 1/10"} + + +def test_sft_status_keeps_existing_metrics_contract(tmp_path) -> None: + run_dir = _run_dir(tmp_path) + expected = { + "task_info": {"total_metrics": 1}, + "metrics": [{"step": 1, "loss": 0.25}], + } + (run_dir / "metrics" / "metrics.json").write_text( + json.dumps(expected), + encoding="utf-8", + ) + + assert get_train_status(str(tmp_path), "task-1", "version-1") == expected + + +def test_train_status_reports_missing_metrics(tmp_path) -> None: + _run_dir(tmp_path) + + with pytest.raises(TaskServiceError) as exc_info: + get_train_status(str(tmp_path), "task-1", "version-1") + + assert exc_info.value.code == 404 diff --git a/tests/test_trainer_no_swanlab.py b/tests/test_trainer_no_swanlab.py new file mode 100644 index 00000000..8fcbad47 --- /dev/null +++ b/tests/test_trainer_no_swanlab.py @@ -0,0 +1,289 @@ +import asyncio +import json +import pickle +import sqlite3 +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest +import yaml + +from api.app.utils.config.config import get_system_config +from loopai.common.db_tool.runtime import get_latest_task_runtime_sync +from loopai.common.tracking import assert_no_retired_tracking +from loopai.skills.Trainer.runner import inspect_prepared_trainer_config +from loopai.skills.Trainer.runtime_config import resolve_trainer_runtime_config +from loopai.skills.Trainer.utils.config_generator import ConfigGenerator +from loopai.skills.Trainer.utils.persistent_worker import load_worker_result +from loopai.skills.Trainer.utils.task_manager import TaskManager +from loopai.skills.Trainer.utils.verl_config_generator import generate_verl_grpo_config +from loopai.skills.Trainer.utils.verl_launcher import load_verl_launch_config + + +def test_generated_sft_yaml_cannot_enable_swanlab(tmp_path) -> None: + dataset_path = tmp_path / "train.json" + dataset_path.write_text("[]\n", encoding="utf-8") + template_path = tmp_path / "legacy-swanlab-template.yaml" + template_path.write_text( + "\n".join( + ( + "stage: sft", + "report_to: swanlab", + "use_swanlab: true", + "swanlab_mode: cloud", + ) + ) + + "\n", + encoding="utf-8", + ) + + generator = ConfigGenerator() + config = generator.generate_config( + task_description="Run a small SFT job.", + dataset_path=str(dataset_path), + model_name="/models/base", + output_dir=str(tmp_path / "output"), + template_path=str(template_path), + ) + generated_path = tmp_path / "training.yaml" + assert generator.save_config_as_yaml(config, str(generated_path)) is True + + generated = yaml.safe_load(generated_path.read_text(encoding="utf-8")) + assert generated["report_to"] == "none" + assert generated.get("use_swanlab") in (None, False) + assert "swanlab_mode" not in generated + assert all( + str(key) == "use_swanlab" and value is False + for key, value in generated.items() + if "swanlab" in str(key).lower() + ) + + +def test_runtime_config_discards_legacy_swanlab_secret(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("SWANLAB_API_KEY", "secret-from-parent-env") + state = { + "task_id": "no-swanlab-runtime", + "output_dir": str(tmp_path / "outputs"), + "system": {"swanlab_api_key": "secret-from-system-state"}, + "trainer": { + "train_framework": "llamafactory", + "train_stage": "sft", + "train_input_dataset_path": str(tmp_path / "train.json"), + "train_input_model_name": "/models/base", + "train_input_task_description": "Run SFT without external tracking.", + "swanlab_api_key": "secret-from-trainer-state", + }, + } + + runtime = resolve_trainer_runtime_config( + state=state, + swanlab_api_key="secret-from-call", + ) + resolved_state = runtime["state"] + + assert "swanlab_api_key" not in resolved_state.get("trainer", {}) + assert "swanlab_api_key" not in resolved_state.get("system", {}) + + +def test_task_manager_does_not_inherit_or_inject_swanlab_secret( + tmp_path, monkeypatch +) -> None: + monkeypatch.setenv("SWANLAB_API_KEY", "secret-from-parent-env") + manager = TaskManager( + configs_dir=str(tmp_path / "configs"), + logs_dir=str(tmp_path / "logs"), + runs_dir=str(tmp_path / "runs"), + app_config={ + "CUDA_VISIBLE_DEVICES": "0", + "swanlab_api_key": "secret-from-app-config", + }, + ) + + try: + child_env = manager._get_safe_env() + finally: + manager.executor.shutdown(wait=False) + + assert "SWANLAB_API_KEY" not in child_env + + +def test_legacy_approved_yaml_is_rejected(tmp_path) -> None: + config_path = tmp_path / "approved.yaml" + config_path.write_text( + "stage: sft\nreport_to: swanlab\nuse_swanlab: true\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="retired SwanLab settings"): + inspect_prepared_trainer_config(str(config_path)) + + +def test_config_api_filters_legacy_system_secret() -> None: + record = SimpleNamespace( + id=1, + name="starter", + config=json.dumps( + { + "system": { + "swanlab_api_key": "legacy-secret", + "CUDA_VISIBLE_DEVICES": "0", + }, + "default_states": {"language": "en"}, + } + ), + ) + + with patch( + "api.app.utils.config.config.check_config_from_db", + new=AsyncMock(return_value=record), + ): + result = asyncio.run(get_system_config("/tmp")) + + assert "swanlab_api_key" not in result["config"] + assert result["config"]["CUDA_VISIBLE_DEVICES"]["value"] == "0" + + +def test_verl_generator_forces_local_loggers(tmp_path) -> None: + template_path = tmp_path / "legacy-verl.yaml" + template_path.write_text( + yaml.safe_dump( + { + "framework": "verl", + "stage": "grpo", + "entrypoint": "verl.trainer.main_ppo", + "environment": {}, + "overrides": {"trainer.logger": ["console", "swanlab"]}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + state = { + "trainer": { + "trainer_output_dir": str(tmp_path / "run"), + "trainer_version_id": "version-1", + "train_input_dataset_path": str(tmp_path / "train.parquet"), + "train_input_eval_dataset_path": str(tmp_path / "eval.parquet"), + "train_input_model_name": "/models/base", + "verl_dir": "/opt/verl", + "verl_env_path": "verl", + "CUDA_VISIBLE_DEVICES": "0", + } + } + + generated = generate_verl_grpo_config(state, str(template_path)) + + assert generated["overrides"]["trainer.logger"] == ["console", "file"] + assert_no_retired_tracking(generated) + + +def test_direct_verl_launcher_rejects_retired_logger(tmp_path) -> None: + config_path = tmp_path / "legacy-verl.yaml" + config_path.write_text( + yaml.safe_dump( + { + "framework": "verl", + "stage": "grpo", + "entrypoint": "verl.trainer.main_ppo", + "environment": {}, + "overrides": {"trainer.logger": ["console", "swanlab"]}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="retired SwanLab settings"): + load_verl_launch_config(str(config_path)) + + +def test_legacy_verl_shell_script_cannot_start_retired_tracker(tmp_path) -> None: + manager = TaskManager( + configs_dir=str(tmp_path / "configs"), + logs_dir=str(tmp_path / "logs"), + runs_dir=str(tmp_path / "runs"), + app_config={}, + ) + script_path = tmp_path / "legacy.sh" + script_path.write_text("python -m swanlab\n", encoding="utf-8") + + try: + with pytest.raises(ValueError, match="retired experiment tracker"): + manager._run_verl_training( + {"status": "running"}, + str(script_path), + str(tmp_path / "train.log"), + object(), + ) + finally: + manager.executor.shutdown(wait=False) + + +def test_legacy_worker_result_is_sanitized_on_load(tmp_path) -> None: + result_path = tmp_path / "worker_result.pkl" + with result_path.open("wb") as file: + pickle.dump( + { + "ok": True, + "state": { + "system": {"swanlab_api_key": "system-secret"}, + "trainer": {"swanlab_api_key": "trainer-secret"}, + }, + }, + file, + ) + + payload = load_worker_result(result_path) + + assert "swanlab_api_key" not in payload["state"]["system"] + assert "swanlab_api_key" not in payload["state"]["trainer"] + with result_path.open("rb") as file: + persisted_payload = pickle.load(file) + assert "swanlab_api_key" not in persisted_payload["state"]["system"] + assert "swanlab_api_key" not in persisted_payload["state"]["trainer"] + + +def test_legacy_task_runtime_state_is_purged_on_read(tmp_path) -> None: + db_path = tmp_path / "runtime.db" + con = sqlite3.connect(db_path) + try: + con.execute( + """ + create table taskruntime( + id integer primary key, + task_id text, + node_name text, + version text, + state text, + status text, + createdAt text, + updatedAt text + ) + """ + ) + con.execute( + """ + insert into taskruntime(task_id, node_name, version, state, status, createdAt, updatedAt) + values(?, ?, ?, ?, ?, datetime('now'), datetime('now')) + """, + ( + "task-1", + "trainer", + "version-1", + json.dumps({"trainer": {"swanlab_api_key": "legacy-secret"}}), + "completed", + ), + ) + con.commit() + finally: + con.close() + + runtime = get_latest_task_runtime_sync(db_path, "task-1", "trainer") + + assert "swanlab_api_key" not in json.loads(runtime["state"])["trainer"] + con = sqlite3.connect(db_path) + try: + persisted_state = con.execute("select state from taskruntime").fetchone()[0] + finally: + con.close() + assert "swanlab_api_key" not in json.loads(persisted_state)["trainer"] diff --git a/tests/test_trainer_persistent_worker.py b/tests/test_trainer_persistent_worker.py new file mode 100644 index 00000000..807b11f1 --- /dev/null +++ b/tests/test_trainer_persistent_worker.py @@ -0,0 +1,274 @@ +import json +import os +import pickle +import threading +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import Mock, patch + +from loopai.skills.Trainer import load_events +from loopai.skills.Trainer.utils.persistent_worker import ( + PersistentWorkerHandle, + launch_or_attach_persistent_worker, + read_run_state, + record_worker_event, + update_run_state, + wait_for_persistent_worker, + worker_paths, + write_worker_result, +) +from loopai.skills.Trainer.worker_entry import run_worker +from loopai.skills.Trainer.utils.training_log_parser import TrainingLogParser + + +def _state(tmp_path: Path) -> dict: + run_dir = tmp_path / "task-1" / "trainer" / "version-1" + return { + "task_id": "task-1", + "output_dir": str(tmp_path), + "trainer": { + "trainer_version_id": "version-1", + "trainer_output_dir": str(run_dir), + "output_dir": str(run_dir), + "train_framework": "verl", + }, + } + + +def test_run_state_updates_are_atomic_and_preserve_worker_status(tmp_path: Path) -> None: + state = _state(tmp_path) + state_path = worker_paths(state["trainer"]["trainer_output_dir"])["state"] + update_run_state(state_path, {"status": "running", "worker_pid": 123}) + + record_worker_event( + state, + { + "node": "training_execution", + "status": "running", + "progress": 0.25, + "message": "training", + "data": { + "current_step": "2", + "total_steps": "8", + "training_pid": 456, + }, + }, + ) + + persisted = read_run_state(state_path) + assert persisted["status"] == "running" + assert persisted["event_sequence"] == 1 + assert persisted["current_step"] == "2" + assert persisted["total_steps"] == "8" + assert persisted["training_pid"] == 456 + + +def test_launcher_uses_detached_worker_process(tmp_path: Path) -> None: + state = _state(tmp_path) + process = Mock(pid=321) + + with patch( + "loopai.skills.Trainer.utils.persistent_worker.subprocess.Popen", + return_value=process, + ) as popen: + handle = launch_or_attach_persistent_worker(state) + + assert handle.process is process + assert popen.call_args.kwargs["start_new_session"] is (os.name == "posix") + persisted = read_run_state(handle.state_path) + assert persisted["status"] == "queued" + assert persisted["worker_pid"] == 321 + assert state["trainer"]["trainer_worker_pid"] == 321 + + +def test_concurrent_attach_launches_only_one_worker(tmp_path: Path) -> None: + first_popen_started = threading.Event() + second_popen_started = threading.Event() + release_first_popen = threading.Event() + process = Mock(pid=321) + + def blocking_popen(*args, **kwargs): + if first_popen_started.is_set(): + second_popen_started.set() + else: + first_popen_started.set() + assert release_first_popen.wait(timeout=5.0) + return process + + with ( + patch( + "loopai.skills.Trainer.utils.persistent_worker.subprocess.Popen", + side_effect=blocking_popen, + ) as popen, + patch( + "loopai.skills.Trainer.utils.persistent_worker._pid_is_alive", + side_effect=lambda pid: int(pid or 0) == 321, + ), + ThreadPoolExecutor(max_workers=2) as executor, + ): + first = executor.submit(launch_or_attach_persistent_worker, _state(tmp_path)) + assert first_popen_started.wait(timeout=5.0) + second = executor.submit(launch_or_attach_persistent_worker, _state(tmp_path)) + assert second_popen_started.wait(timeout=0.5) is False + assert popen.call_count == 1 + release_first_popen.set() + first_handle = first.result(timeout=5.0) + second_handle = second.result(timeout=5.0) + + assert popen.call_count == 1 + assert first_handle.process is process + assert second_handle.process is None + assert read_run_state(first_handle.state_path)["worker_pid"] == 321 + + +def test_completed_worker_result_is_reused_without_relaunch(tmp_path: Path) -> None: + state = _state(tmp_path) + paths = worker_paths(state["trainer"]["trainer_output_dir"]) + write_worker_result(paths["result"], {"ok": True, "state": state}) + + with patch("loopai.skills.Trainer.utils.persistent_worker.subprocess.Popen") as popen: + handle = launch_or_attach_persistent_worker(state) + + assert handle.result_path == paths["result"] + popen.assert_not_called() + + +def test_verl_progress_prefers_latest_structured_metric(tmp_path: Path) -> None: + log_path = tmp_path / "train.log" + metrics_path = tmp_path / "verl_metrics.jsonl" + log_path.write_text( + "Training Progress: 2%|▏ | 1/58 [00:01<10:00, 10.00s/it]\r" + "Training Progress: 16%|█▌ | 9/58 [02:00<08:00, 10.00s/it]\n", + encoding="utf-8", + ) + metrics_path.write_text( + "\n".join( + [ + json.dumps({"step": 20, "data": {"training/global_step": 20}}), + json.dumps({"step": 21, "data": {"training/global_step": 21}}), + ] + ), + encoding="utf-8", + ) + + progress = TrainingLogParser().parse_verl_metrics_progress( + str(log_path), + str(metrics_path), + ) + + assert progress["progress_text"] == "21/58" + assert progress["time_text"] == "Verl metrics" + + +def test_tqdm_parser_uses_last_carriage_return_refresh(tmp_path: Path) -> None: + log_path = tmp_path / "train.log" + log_path.write_text( + "Training Progress: | 1/58 [00:01<10:00, 10.00s/it]\r" + "Training Progress: | 9/58 [02:00<08:00, 10.00s/it]\n", + encoding="utf-8", + ) + + progress = TrainingLogParser().parse_training_progress(str(log_path)) + + assert progress["progress_text"] == "9/58" + + +def test_worker_entry_persists_result_and_terminal_state(tmp_path: Path) -> None: + state = _state(tmp_path) + paths = worker_paths(state["trainer"]["trainer_output_dir"]) + paths["run_dir"].mkdir(parents=True, exist_ok=True) + with paths["request"].open("wb") as request_file: + pickle.dump({"state": state}, request_file) + + def fake_training(worker_state, writer=None): + worker_state["trainer"]["trainer_training_success"] = True + worker_state["trainer"]["trainer_training_final_status"] = {"status": "completed"} + return worker_state + + with ( + patch( + "loopai.skills.Trainer.nodes.training_execution_node._training_execution_inline", + side_effect=fake_training, + ), + patch("loopai.skills.Trainer.runner.finalize_trainer_result_state", side_effect=lambda value, **_: value), + patch("loopai.skills.Trainer.utils.stream_events.emit_trainer_event"), + ): + assert run_worker(paths["request"]) == 0 + + assert not paths["request"].exists() + persisted = read_run_state(paths["state"]) + assert persisted["status"] == "completed" + with paths["result"].open("rb") as result_file: + result = pickle.load(result_file) + assert result["ok"] is True + assert result["state"]["trainer"]["trainer_training_success"] is True + + +def test_worker_entry_preserves_cancelled_terminal_state(tmp_path: Path) -> None: + state = _state(tmp_path) + state["trainer"]["_trainer_worker_runtime"] = { + "task_state_loaded": True, + "thread_id": "task-1", + "db_path": str(tmp_path / "task.db"), + } + paths = worker_paths(state["trainer"]["trainer_output_dir"]) + paths["run_dir"].mkdir(parents=True, exist_ok=True) + with paths["request"].open("wb") as request_file: + pickle.dump({"state": state}, request_file) + + def fake_cancelled_training(worker_state, writer=None): + worker_state["trainer"]["trainer_training_success"] = False + worker_state["trainer"]["trainer_training_final_status"] = {"status": "cancelled"} + worker_state["trainer"]["train_output_training_error"] = "cancelled by user" + return worker_state + + persisted = {} + + def fake_state_update(runtime, trainer_state): + persisted["runtime"] = runtime + persisted["result"] = trainer_state.get("trainer_result") + persisted["last_error"] = trainer_state.get("trainer_last_error") + + with ( + patch( + "loopai.skills.Trainer.nodes.training_execution_node._training_execution_inline", + side_effect=fake_cancelled_training, + ), + patch("loopai.skills.Trainer.runner.finalize_trainer_result_state", side_effect=lambda value, **_: value), + patch("loopai.skills.Trainer.runner._update_trainer_task_state", side_effect=fake_state_update), + patch("loopai.skills.Trainer.utils.stream_events.emit_trainer_event"), + ): + assert run_worker(paths["request"]) == 0 + + assert read_run_state(paths["state"])["status"] == "cancelled" + result = wait_for_persistent_worker( + PersistentWorkerHandle( + result_path=paths["result"], + state_path=paths["state"], + run_dir=paths["run_dir"], + ), + poll_interval=0.01, + ) + assert result["trainer"]["trainer_training_final_status"]["status"] == "cancelled" + assert result["trainer"]["trainer_result"]["ok"] is False + assert result["trainer"]["trainer_result"]["error"]["code"] == "INTERRUPTED" + assert persisted["result"]["ok"] is False + assert persisted["last_error"]["code"] == "INTERRUPTED" + + +def test_real_detached_worker_writes_failure_result_without_training(tmp_path: Path) -> None: + state = _state(tmp_path) + state["trainer"]["trainer_config_generation_success"] = False + + handle = launch_or_attach_persistent_worker(state) + result = wait_for_persistent_worker(handle, poll_interval=0.05) + + assert result["trainer"]["trainer_training_success"] is False + assert "配置生成未成功" in result["trainer"]["train_output_training_error"] + assert result["trainer"]["trainer_result"]["ok"] is False + assert "配置生成未成功" in result["trainer"]["trainer_last_error"]["detail"] + persisted = read_run_state(handle.state_path) + assert persisted["status"] == "failed" + assert handle.result_path.is_file() + events = load_events(task_id="task-1", output_dir=str(tmp_path), version_id="version-1") + assert events[-1]["status"] == "failed" diff --git a/tests/test_trainer_task_lifecycle.py b/tests/test_trainer_task_lifecycle.py index 8e7c2418..84e940b2 100644 --- a/tests/test_trainer_task_lifecycle.py +++ b/tests/test_trainer_task_lifecycle.py @@ -3,7 +3,10 @@ import signal from unittest.mock import Mock, patch -from loopai.skills.Trainer.nodes.training_execution_node import training_execution_node +from loopai.skills.Trainer.nodes.training_execution_node import ( + _training_execution_inline, + training_execution_node, +) from loopai.skills.Trainer.utils.task_manager import TaskManager from loopai.skills.Trainer.utils.task_status import TaskStatus @@ -19,6 +22,8 @@ def _manager(tmp_path) -> TaskManager: def test_llamafactory_uses_dedicated_process_group(tmp_path) -> None: manager = _manager(tmp_path) + config_path = tmp_path / "train.yaml" + config_path.write_text("report_to: none\n", encoding="utf-8") task_info = { "status": TaskStatus.RUNNING, "process": None, @@ -35,7 +40,7 @@ def test_llamafactory_uses_dedicated_process_group(tmp_path) -> None: ) as popen: manager._run_llamafactory_training( task_info, - str(tmp_path / "train.yaml"), + str(config_path), str(tmp_path / "train.log"), log_parser, ) @@ -80,7 +85,10 @@ def test_wait_for_completion_joins_training_future(tmp_path) -> None: def test_training_node_has_no_fixed_one_hour_early_exit() -> None: - source = inspect.getsource(training_execution_node) + source = inspect.getsource(_training_execution_inline) assert "max_wait_time" not in source assert "wait_for_completion" in source + + wrapper_source = inspect.getsource(training_execution_node) + assert "wait_for_persistent_worker" in wrapper_source diff --git a/tests/test_trainer_verl_integration.py b/tests/test_trainer_verl_integration.py new file mode 100644 index 00000000..8bd25e11 --- /dev/null +++ b/tests/test_trainer_verl_integration.py @@ -0,0 +1,469 @@ +import json +import sys +from types import ModuleType, SimpleNamespace +from pathlib import Path +from unittest.mock import Mock, patch + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +import yaml + +from loopai.skills.Trainer import prepare +from loopai.skills.Trainer.results import analyze_results, select_best_checkpoint +from loopai.skills.Trainer.rewards import compute_score, normalize_reward_preset +from loopai.skills.Trainer.runtime_config import ( + _DEFAULT_VERL_GRPO_TEMPLATE_PATH, + resolve_trainer_runtime_config, +) +from loopai.skills.Trainer.utils.realtime_log_parser import MetricsExtractor +from loopai.skills.Trainer.utils.task_manager import TaskManager +from loopai.skills.Trainer.utils.task_status import TaskStatus +from loopai.skills.Trainer.utils.verl_config_generator import generate_verl_grpo_config +from loopai.skills.Trainer.utils.verl_data_checker import check_verl_grpo_inputs +from loopai.skills.Trainer.utils.verl_exporter import export_verl_checkpoint +from loopai.skills.Trainer.utils.verl_launcher import build_verl_launch + + +_VERL_GRPO_SMOKE_TEMPLATE_PATH = ( + Path(__file__).resolve().parents[1] + / "loopai" + / "skills" + / "Trainer" + / "templates" + / "verl_grpo_smoke.yaml" +) + + +def _write_rl_parquet(path: Path) -> None: + table = pa.table({ + "prompt": [[{"role": "user", "content": "2+2?"}]], + "data_source": ["unit_test"], + "reward_model": [{"ground_truth": "4"}], + "extra_info": [{"index": 0}], + }) + pq.write_table(table, path) + + +def _verl_state(tmp_path: Path) -> dict: + tmp_path.mkdir(parents=True, exist_ok=True) + train_path = tmp_path / "train.parquet" + validation_path = tmp_path / "validation.parquet" + _write_rl_parquet(train_path) + _write_rl_parquet(validation_path) + reward_path = tmp_path / "reward.py" + reward_path.write_text("def compute_score(data_source, solution_str, ground_truth, extra_info=None):\n return 1.0\n") + verl_dir = tmp_path / "verl-repo" + (verl_dir / "verl" / "trainer").mkdir(parents=True) + return { + "task_id": "verl-task", + "output_dir": str(tmp_path / "outputs"), + "trainer": { + "train_framework": "verl", + "train_stage": "grpo", + "verl_dir": str(verl_dir), + "train_input_dataset_path": str(train_path), + "train_input_eval_dataset_path": str(validation_path), + "train_input_model_name": "/models/base/", + "train_input_task_description": "Optimize unit-test reward with GRPO.", + "verl_reward_function_path": str(reward_path), + "CUDA_VISIBLE_DEVICES": "0,1", + }, + } + + +def test_runtime_routes_sft_and_grpo_to_only_supported_backends(tmp_path: Path) -> None: + runtime = resolve_trainer_runtime_config(state=_verl_state(tmp_path)) + trainer = runtime["state"]["trainer"] + assert runtime["missing_fields"] == [] + assert trainer["train_input_config_template_path"] == str(_DEFAULT_VERL_GRPO_TEMPLATE_PATH) + assert trainer["verl_env_path"] == "verl" + + bad_state = _verl_state(tmp_path / "bad") + bad_state["trainer"]["train_framework"] = "llamafactory" + with pytest.raises(ValueError, match="backend/stage combination"): + resolve_trainer_runtime_config(state=bad_state) + + +def test_verl_data_and_reward_preflight(tmp_path: Path) -> None: + state = _verl_state(tmp_path) + trainer = state["trainer"] + result = check_verl_grpo_inputs( + trainer["train_input_dataset_path"], + trainer["train_input_eval_dataset_path"], + trainer["verl_reward_function_path"], + "compute_score", + ) + assert result["is_valid"] is True + assert result["train"]["rows"] == 1 + assert {"prompt", "data_source", "reward_model"}.issubset(result["train"]["columns"]) + + +def test_named_reward_preset_accepts_custom_data_source_and_auto_rejects_it(tmp_path: Path) -> None: + state = _verl_state(tmp_path) + trainer = state["trainer"] + + preset_result = check_verl_grpo_inputs( + trainer["train_input_dataset_path"], + trainer["train_input_eval_dataset_path"], + reward_mode="preset", + reward_preset="math_boxed", + ) + assert preset_result["is_valid"] is True + assert preset_result["reward"]["preset"] == "math_boxed" + assert preset_result["train"]["semantic_sample_size"] == 1 + + auto_result = check_verl_grpo_inputs( + trainer["train_input_dataset_path"], + trainer["train_input_eval_dataset_path"], + reward_mode="auto", + ) + assert auto_result["is_valid"] is False + assert "unit_test" in auto_result["errors"][0] + + +def test_verl_data_preflight_rejects_missing_ground_truth(tmp_path: Path) -> None: + train_path = tmp_path / "train.parquet" + validation_path = tmp_path / "validation.parquet" + invalid = pa.table({ + "prompt": [[{"role": "user", "content": "2+2?"}]], + "data_source": ["unit_test"], + "reward_model": [{"ground_truth": None}], + }) + pq.write_table(invalid, train_path) + pq.write_table(invalid, validation_path) + + result = check_verl_grpo_inputs( + str(train_path), + str(validation_path), + reward_mode="preset", + reward_preset="math_boxed", + ) + assert result["is_valid"] is False + assert any("reward_model.ground_truth" in error for error in result["errors"]) + + +def test_reward_router_uses_verl_implementation_without_copying_it() -> None: + reward_score = ModuleType("verl.utils.reward_score") + reward_score.math_reward = SimpleNamespace( + compute_score=lambda solution_str, ground_truth: solution_str == ground_truth + ) + verl_module = ModuleType("verl") + utils_module = ModuleType("verl.utils") + + with patch.dict( + sys.modules, + { + "verl": verl_module, + "verl.utils": utils_module, + "verl.utils.reward_score": reward_score, + }, + ): + assert compute_score("any", "4", "4", preset="math") == 1.0 + + assert normalize_reward_preset("qa-em") == "qa_exact_match" + + +def test_generated_yaml_uses_loopai_reward_preset_and_kwargs(tmp_path: Path) -> None: + state = _verl_state(tmp_path) + trainer = state["trainer"] + trainer.pop("verl_reward_function_path") + trainer["verl_reward_mode"] = "preset" + trainer["verl_reward_preset"] = "gsm8k_exact" + trainer["verl_reward_kwargs"] = {"method": "flexible"} + + runtime = resolve_trainer_runtime_config(state=state) + config = generate_verl_grpo_config( + runtime["state"], + str(_DEFAULT_VERL_GRPO_TEMPLATE_PATH), + ) + overrides = config["overrides"] + + assert config["loopai_reward"]["mode"] == "preset" + assert config["loopai_reward"]["preset"] == "gsm8k_exact" + assert overrides["reward.custom_reward_function.path"].endswith( + "loopai/skills/Trainer/rewards/router.py" + ) + assert overrides["reward.custom_reward_function.name"] == "compute_score" + assert overrides["+reward.custom_reward_function.reward_kwargs"] == { + "method": "flexible", + "preset": "gsm8k_exact", + } + + config_path = tmp_path / "preset-launch.yaml" + config_path.write_text( + __import__("yaml").safe_dump(config, sort_keys=False), + encoding="utf-8", + ) + with patch("loopai.skills.Trainer.utils.verl_launcher.shutil.which", return_value="/conda"): + command, _, _ = build_verl_launch(str(config_path)) + assert ( + '+reward.custom_reward_function.reward_kwargs={method:"flexible",preset:"gsm8k_exact"}' + in command + ) + + +def test_prepare_generates_approvable_verl_yaml_without_training(tmp_path: Path, monkeypatch) -> None: + monkeypatch.delenv("TASK_ID", raising=False) + monkeypatch.delenv("DB_PATH", raising=False) + result = prepare(state=_verl_state(tmp_path), thread_id="verl-task") + approval = result["trainer"]["trainer_result"]["data"] + config = approval["config"] + + assert approval["approval_required"] is True + assert config["framework"] == "verl" + assert config["stage"] == "grpo" + assert config["environment"]["verl_env_path"] == "verl" + assert config["overrides"]["algorithm.adv_estimator"] == "grpo" + assert config["overrides"]["actor_rollout_ref.model.path"] == "/models/base" + assert config["overrides"]["actor_rollout_ref.actor.use_dynamic_bsz"] is True + assert config["overrides"]["actor_rollout_ref.rollout.log_prob_use_dynamic_bsz"] is True + assert config["overrides"]["trainer.n_gpus_per_node"] == 2 + assert config["overrides"]["trainer.max_actor_ckpt_to_keep"] == 10 + + +def test_smoke_template_limits_work_and_supports_eight_gpus(tmp_path: Path) -> None: + state = _verl_state(tmp_path) + state["trainer"]["CUDA_VISIBLE_DEVICES"] = "0,1,2,3,4,5,6,7" + state["trainer"]["verl_max_actor_ckpt_to_keep"] = 1 + config = generate_verl_grpo_config(state, str(_VERL_GRPO_SMOKE_TEMPLATE_PATH)) + overrides = config["overrides"] + + assert overrides["data.train_batch_size"] == 8 + assert overrides["data.train_max_samples"] == 80 + assert overrides["data.val_max_samples"] == 8 + assert overrides["actor_rollout_ref.rollout.n"] == 2 + assert overrides["actor_rollout_ref.actor.use_torch_compile"] is False + assert overrides["trainer.total_training_steps"] == 10 + assert overrides["trainer.test_freq"] == 1 + assert overrides["trainer.n_gpus_per_node"] == 8 + assert overrides["trainer.max_actor_ckpt_to_keep"] == 1 + assert config["result"]["export_huggingface"] is False + + +def test_production_template_saves_at_validation_cadence() -> None: + config = yaml.safe_load(Path(_DEFAULT_VERL_GRPO_TEMPLATE_PATH).read_text(encoding="utf-8")) + overrides = config["overrides"] + assert overrides["trainer.save_freq"] > 0 + assert overrides["trainer.save_freq"] == overrides["trainer.test_freq"] + assert overrides["trainer.max_actor_ckpt_to_keep"] == 10 + + +def test_launcher_uses_conda_verl_and_no_shell(tmp_path: Path) -> None: + state = _verl_state(tmp_path) + state["trainer"]["trainer_output_dir"] = str(tmp_path / "run") + config = generate_verl_grpo_config(state, str(_DEFAULT_VERL_GRPO_TEMPLATE_PATH)) + config_path = tmp_path / "launch.yaml" + config_path.write_text(__import__("yaml").safe_dump(config, sort_keys=False), encoding="utf-8") + + with patch("loopai.skills.Trainer.utils.verl_launcher.shutil.which", return_value="/conda"): + cmd, cwd, env = build_verl_launch(str(config_path)) + + assert cmd[:7] == ["/conda", "run", "--no-capture-output", "-n", "verl", "python", "-m"] + assert cmd[7] == "verl.trainer.main_ppo" + assert any(item == "algorithm.adv_estimator=\"grpo\"" for item in cmd) + assert cwd == state["trainer"]["verl_dir"] + assert env["VERL_FILE_LOGGER_PATH"].endswith("metrics/verl_metrics.jsonl") + + +def test_verl_console_metrics_are_normalized() -> None: + metrics = MetricsExtractor().extract_metrics( + "step:10 - critic/rewards/mean:0.75 - actor/pg_loss:-0.12 - " + "val-core/unit_test/acc/mean@8:0.8 - training/global_step:10" + ) + assert metrics["step"] == 10 + assert metrics["reward"] == 0.75 + assert metrics["policy_loss"] == -0.12 + assert metrics["val_score"] == 0.8 + + +def test_results_select_verl_global_step_by_validation_metric(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + metrics_dir = run_dir / "metrics" + checkpoints_dir = run_dir / "checkpoints" + metrics_dir.mkdir(parents=True) + for step in (5, 10): + actor_dir = checkpoints_dir / f"global_step_{step}" / "actor" + actor_dir.mkdir(parents=True) + (actor_dir / "model_world_size_2_rank_0.pt").write_bytes(b"checkpoint") + metric_name = "val-core/unit_test/acc/mean@8" + with (metrics_dir / "verl_metrics.jsonl").open("w", encoding="utf-8") as stream: + stream.write(json.dumps({"step": 5, "data": {metric_name: 0.2}}) + "\n") + stream.write(json.dumps({"step": 10, "data": {metric_name: 0.8}}) + "\n") + + payload = analyze_results( + training_output_dir=str(run_dir), + trainer_state={ + "verl_selection_metric": "val-core/*/acc/mean@*", + "verl_selection_mode": "max", + "train_config": {"overrides": {"trainer.default_local_dir": str(checkpoints_dir)}}, + }, + ) + best = payload["data"]["best_checkpoint"] + assert best["name"] == "global_step_10" + assert best["metric"]["value"] == 0.8 + assert best["needs_merge"] is True + + +def test_approved_yaml_selection_metric_overrides_runtime_default(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + metrics_dir = run_dir / "metrics" + checkpoints_dir = run_dir / "checkpoints" + metrics_dir.mkdir(parents=True) + for step in (5, 10): + (checkpoints_dir / f"global_step_{step}" / "actor").mkdir(parents=True) + with (metrics_dir / "verl_metrics.jsonl").open("w", encoding="utf-8") as stream: + stream.write(json.dumps({ + "step": 5, + "data": { + "approved/score": 0.9, + "val-core/unit_test/acc/mean@8": 0.2, + }, + }) + "\n") + stream.write(json.dumps({ + "step": 10, + "data": { + "approved/score": 0.1, + "val-core/unit_test/acc/mean@8": 0.8, + }, + }) + "\n") + + payload = analyze_results( + training_output_dir=str(run_dir), + trainer_state={ + "verl_selection_metric": "val-core/*/acc/mean@*", + "verl_selection_mode": "max", + "train_config": { + "overrides": {"trainer.default_local_dir": str(checkpoints_dir)}, + "result": { + "selection_metric": "approved/score", + "selection_mode": "max", + }, + }, + }, + ) + + assert payload["data"]["best_checkpoint"]["name"] == "global_step_5" + + +def test_metric_step_uses_nearest_previous_checkpoint() -> None: + checkpoints = [ + {"name": "global_step_5", "step": 5, "path": "/ckpt/5"}, + {"name": "global_step_10", "step": 10, "path": "/ckpt/10"}, + ] + best = select_best_checkpoint( + checkpoints, + [{"step": 8, "val_score": 0.9}], + selection_metric="val_score", + selection_mode="max", + ) + assert best["name"] == "global_step_5" + assert "not after" in best["selection_reason"] + + +def test_checkpoint_selection_ignores_non_finite_metrics() -> None: + checkpoints = [ + {"name": "global_step_5", "step": 5, "path": "/ckpt/5"}, + {"name": "global_step_10", "step": 10, "path": "/ckpt/10"}, + ] + best = select_best_checkpoint( + checkpoints, + [ + {"step": 5, "val_score": float("nan")}, + {"step": 8, "val_score": float("inf")}, + {"step": 10, "val_score": 0.5}, + ], + selection_metric="val_score", + selection_mode="max", + ) + assert best["name"] == "global_step_10" + assert best["metric"]["value"] == 0.5 + + +def test_invalid_checkpoint_selection_mode_is_rejected(tmp_path: Path) -> None: + checkpoints = [{"name": "global_step_1", "step": 1, "path": "/ckpt/1"}] + with pytest.raises(ValueError, match="selection_mode must be max or min"): + select_best_checkpoint( + checkpoints, + [{"step": 1, "val_score": 0.5}], + selection_metric="val_score", + selection_mode="maximize", + ) + + state = _verl_state(tmp_path) + config = generate_verl_grpo_config(state, str(_DEFAULT_VERL_GRPO_TEMPLATE_PATH)) + config["result"]["selection_mode"] = "maximize" + config_path = tmp_path / "invalid-selection.yaml" + config_path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + with pytest.raises(ValueError, match="result.selection_mode must be max or min"): + build_verl_launch(str(config_path)) + + +def test_task_manager_launches_verl_in_dedicated_process_group(tmp_path: Path) -> None: + manager = TaskManager( + configs_dir=str(tmp_path / "configs"), + logs_dir=str(tmp_path / "logs"), + runs_dir=str(tmp_path / "runs"), + app_config={"verl_dir": str(tmp_path)}, + ) + task_info = { + "status": TaskStatus.RUNNING, + "process": None, + "process_group_id": None, + "cancel_requested": False, + } + process = Mock(pid=43210) + process.wait.return_value = 0 + log_parser = Mock() + launch = (["conda", "run", "-n", "verl", "python", "-m", "verl.trainer.main_ppo"], str(tmp_path), {}) + + with patch("loopai.skills.Trainer.utils.task_manager.build_verl_launch", return_value=launch), patch( + "loopai.skills.Trainer.utils.task_manager.subprocess.Popen", return_value=process + ) as popen: + manager._run_verl_training( + task_info, + str(tmp_path / "train.yaml"), + str(tmp_path / "train.log"), + log_parser, + ) + + assert popen.call_args.kwargs["start_new_session"] is (__import__("os").name == "posix") + log_parser.start_monitoring.assert_called_once_with() + assert task_info["status"] == TaskStatus.COMPLETED + + +def test_selected_verl_checkpoint_is_exported_with_same_conda_environment(tmp_path: Path) -> None: + checkpoint_root = tmp_path / "global_step_10" + actor_dir = checkpoint_root / "actor" + actor_dir.mkdir(parents=True) + config_path = tmp_path / "train.yaml" + config_path.write_text("framework: verl\nstage: grpo\n", encoding="utf-8") + launch = ( + ["/conda", "run", "--no-capture-output", "-n", "verl", "python", "-m", "verl.trainer.main_ppo"], + str(tmp_path), + {}, + ) + + def fake_run(cmd, **kwargs): + target_dir = Path(cmd[cmd.index("--target_dir") + 1]) + target_dir.mkdir(parents=True) + (target_dir / "model.safetensors").write_bytes(b"weights") + return Mock(returncode=0) + + with patch( + "loopai.skills.Trainer.utils.verl_exporter.build_verl_launch", + return_value=launch, + ), patch( + "loopai.skills.Trainer.utils.verl_exporter.subprocess.run", + side_effect=fake_run, + ) as run: + output = export_verl_checkpoint( + str(config_path), + {"checkpoint_path": str(checkpoint_root)}, + str(tmp_path / "export.log"), + ) + + cmd = run.call_args.args[0] + assert cmd[:7] == launch[0][:7] + assert cmd[7:11] == ["verl.model_merger", "merge", "--backend", "fsdp"] + assert output.endswith("global_step_10/merged_huggingface") diff --git a/tutorial/docs/guide/cli-tutorial.md b/tutorial/docs/guide/cli-tutorial.md index 5e808498..e414fde1 100644 --- a/tutorial/docs/guide/cli-tutorial.md +++ b/tutorial/docs/guide/cli-tutorial.md @@ -53,8 +53,9 @@ python examples/scripts/run_trainer.py 可选但强烈建议给到的字段： - `output_dir`：Trainer 的工作目录，默认会落到 `./output/trainer` -- `train_input_use_swanlab` / `train_input_swanlab_project`：开启 SwanLab 监控 -- `swanlab_api_key` / `llamafactory_env_path` / `CUDA_VISIBLE_DEVICES`：未在 `state.system` 中提供时需在 `state.trainer` 直接给出 +- `llamafactory_env_path` / `CUDA_VISIBLE_DEVICES`：未在 `state.system` 中提供时需在 `state.trainer` 直接给出 + +Trainer 的进度、曲线和结果分析由本地指标文件驱动，不需要配置外部实验跟踪服务。 > 完整字段含义见 [Trainer Agent 详细指南](/guide/details/trainer-agent#输入字段表-state-trainer)。 @@ -74,10 +75,7 @@ training_state = { "train_input_model_name": "/your/path/to/Qwen2.5-Coder-7B", "output_dir": "./output/trainer_demo", - "train_input_use_swanlab": True, - "train_input_swanlab_project": "sql_sft", # 单独运行时如果 starter.yaml 没加载到 system，可在这里直接补 - # "swanlab_api_key": "xxx", # "llamafactory_env_path": "/opt/conda/envs/llamafactory", # "CUDA_VISIBLE_DEVICES": "0,1", } @@ -90,9 +88,9 @@ training_state = { - **只想跑数据检查**：`from loopai.skills.Trainer.utils.data_checker import check_data_format, generate_format_report`，传入数据集路径即可，不依赖 LangGraph。 - **复跑某次任务**：`thread_id` 用同一个值（脚本里默认是 `"trainer_test_1"`），LangGraph 会从 checkpointer 里恢复上次状态；想要从 0 开始就换一个新的 `thread_id`。 - **想看到完整 SSE 流**：把 `graph.invoke(...)` 换成 `for event in graph.stream(training_state, config=config, stream_mode="custom"): print(event)`，会逐条打出 `StreamEvent`。 -- **长时间训练**：Trainer 没有固定的 1 小时等待上限，会保持前台同步直到训练完成、失败或取消；调用方中断时会取消训练进程组，避免遗留 torchrun/rank 子进程。 +- **长时间训练**：Trainer 没有固定的 1 小时等待上限。调用方正常存活时会同步等待独立 Trainer Worker 到达完成、失败或取消状态；调用方意外中断时 Worker 会继续训练，可通过同一 `task_id`、`version_id` 以及运行目录中的 `run_state.json`/`worker_result.pkl` 重新读取进度和结果。 - **训练失败定位**：先看 `result["trainer"]["train_output_training_log_path"]` 指向的日志文件（LlamaFactory 子进程原始 stdout/stderr），再看 `train_output_training_report_path` 中的汇总报告。 -- **SwanLab 日志看不到**：确认模板 `report_to` 没被覆盖成 `none`，并且 `swanlab_api_key` 能从 `state.trainer` 或 `state.system` 之一读到。 +- **曲线或指标未更新**：SFT 检查运行目录中的 `trainer_log.jsonl` 和 `metrics/metrics.json`；Verl 检查 `metrics/verl_metrics.jsonl`，同时确认训练子进程仍在写日志。 #### 输出位置 @@ -102,5 +100,6 @@ training_state = { - `training_config.yaml` + `config_explanation.txt`：自动生成的 LlamaFactory 配置与说明 - `configs/{trainer_task_id}.yaml`：实际提交给训练子进程的配置副本 - `logs/`、`runs/`：TaskManager 维护的训练日志与运行记录 +- `metrics/metrics.json`（SFT）或 `metrics/verl_metrics.jsonl`（Verl）：前端曲线和结果分析使用的本地指标 - `training_log_{task_id}.txt`、`training_report_{task_id}.txt`：最终汇总日志与报告 - 训练产物（`checkpoint-*`、`trainer_log.jsonl`）写在配置里的 `output_dir`，可在 `result["trainer"]["training_checkpoints"]` 里拿到列表 diff --git a/tutorial/docs/guide/details/trainer-agent.md b/tutorial/docs/guide/details/trainer-agent.md index 3abd6b72..5d2374b9 100644 --- a/tutorial/docs/guide/details/trainer-agent.md +++ b/tutorial/docs/guide/details/trainer-agent.md @@ -1,8 +1,8 @@ # Trainer Agent 详细指南 -`TrainerAgent` 是 LoopAI 闭环里负责模型更新的节点。它会把前面 `Analyzer`、`Obtainer`、`Constructor` 准备好的训练数据，转化为一次完整的微调任务，并将训练日志、checkpoint、SwanLab 监控数据等写回 `state`，供下一轮 `Judger` 使用。 +`TrainerAgent` 是 LoopAI 闭环里负责模型更新的节点。它会把前面 `Analyzer`、`Obtainer`、`Constructor` 准备好的训练数据，转化为一次完整的微调任务，并将本地训练指标、checkpoint 等写回 `state`，供下一轮 `Judger` 使用。 -当前主路径基于 [LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory) 执行 SFT。虽然代码中为其他训练路径预留了扩展分支，但当前文档仍以 LLaMA-Factory 为主进行说明。 +当前支持两条严格配对的训练路径：使用 [LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory) 执行 SFT，以及使用 Verl 执行 GRPO。两条路径共享配置审批、版本化状态、持久化 Worker 和本地指标接口，但数据格式、训练 YAML、checkpoint 形式与结果选择规则分别处理。 ## 在闭环中的位置 @@ -22,7 +22,7 @@ Trainer 的输出通常包括： - 数据检查报告 - 自动生成的 YAML 训练配置和配置说明文本 - 训练日志、训练报告、`trainer_log.jsonl` 解析出的 step-loss -- SwanLab 本地日志路径 +- `metrics/metrics.json` 等本地指标文件 - 训练产生的 `checkpoint-*` 目录列表 ## 执行流程 @@ -141,7 +141,7 @@ LLM 辅助模式下，如果 `ConfigGenerator` 初始化时提供了 `model_path 2. 将生成的 YAML 配置复制到 `{output_dir}/configs/{trainer_task_id}.yaml` 3. 通过 `TaskManager.start_training` 启动训练子进程，并每 30 秒轮询一次任务状态，直到训练完成、失败或取消 4. 训练过程中实时解析 LLaMA-Factory 的日志，将 step、总 step、训练时间等信息写回 `state` 和 SSE 流 -5. 训练结束后扫描 `output_dir`，汇总 `checkpoint-*` 目录、解析 `trainer_log.jsonl` 中的 step-loss、获取 SwanLab 本地日志路径，并生成训练报告 +5. 训练结束后扫描 `output_dir`，汇总 `checkpoint-*` 目录、解析 `trainer_log.jsonl` 和 `metrics/metrics.json` 中的训练指标，并生成训练报告 ## 输入字段表：`state.trainer` @@ -163,9 +163,6 @@ LLM 辅助模式下，如果 `ConfigGenerator` 初始化时提供了 `model_path | 字段名 | 类型 | 默认值 | 说明 | | --- | --- | --- | --- | | `output_dir` | `str` | `./output/trainer` | Trainer 工作目录。最终路径通常为 `{global_output_dir}/{global_task_id}/trainer/{trainer_task_id}`。 | -| `train_input_use_swanlab` | `bool` | `True` | 是否启用 SwanLab 监控，对应 `report_to: swanlab`。 | -| `train_input_swanlab_project` | `str` | `llamafactory_training` | SwanLab 项目名。 | -| `swanlab_api_key` | `str` | 来自 `state.system` | SwanLab 鉴权 key。 | | `llamafactory_env_path` | `str` | 来自 `state.system` | LLaMA-Factory 所在 Conda / venv 路径。 | | `CUDA_VISIBLE_DEVICES` | `str` | `0,1` | 训练进程可见 GPU。 | @@ -187,7 +184,6 @@ LLM 辅助模式下，如果 `ConfigGenerator` 初始化时提供了 `model_path | `trainer_training_final_status` | `dict` | 形如 `{status, created_at, started_at, completed_at, error_message}` 的最终状态字典。 | | `train_output_training_log_path` | `str` | 训练日志保存路径。 | | `train_output_training_report_path` | `str` | 训练报告路径。 | -| `train_output_swanlab_log_path` | `str` | SwanLab 本地日志路径。 | | `training_checkpoints` | `List[str]` | 产出的 checkpoint 目录列表。 | | `training_step_losses` | `List[Dict]` | 从 `trainer_log.jsonl` 解析出的 step-loss。 | @@ -261,8 +257,6 @@ state = { "loopai/skills/Trainer/templates/qwen2_5_coder_bird_full_sft.yaml", "train_input_model_name": "/path/to/Qwen2.5-1.5B", "output_dir": "./output/trainer_demo", - "train_input_use_swanlab": True, - "train_input_swanlab_project": "demo_llamafactory_training", } } @@ -279,7 +273,7 @@ print(summary["final_status"], summary["output_files"]) 在 WebUI 中使用 Trainer 时，通常建议： -1. 先在 `Config` 面板中配置 `system` 级别的 `llamafactory_dir`、`llamafactory_env_path`、`CUDA_VISIBLE_DEVICES`、`swanlab_api_key` +1. 先在 `Config` 面板中配置 `system` 级别的 `llamafactory_dir`、`llamafactory_env_path`、`CUDA_VISIBLE_DEVICES` 2. 在资源池中维护好以下三类路径，再在任务面板中下拉选用 3. 通过对话提供 `train_input_task_description`，让规则模式或 LLM 模式自动决定关键超参 @@ -293,7 +287,7 @@ print(summary["final_status"], summary["output_files"]) - LLaMA-Factory 主仓库需要能正常运行 `llamafactory-cli train` - `llamafactory_env_path` 指向的 Python 环境需要安装 LLaMA-Factory 及其训练依赖，如 `deepspeed`、`transformers` -- 启用 SwanLab 时，需要安装 `swanlab` 并配置 `swanlab_api_key` +- 训练指标由本地文件驱动，不需要安装或配置外部实验跟踪服务 - 多卡训练通过 `CUDA_VISIBLE_DEVICES` 控制，例如 `"0,1,2,3"` ## 常见问题 @@ -301,8 +295,8 @@ print(summary["final_status"], summary["output_files"]) - 数据检查未通过：`.json` 顶层必须是 `list`，`.jsonl` 每行必须是合法 JSON，`conversations[*].from` 必须是 `human/gpt/system` 之一 - 配置生成失败：通常是模板路径不存在，或 YAML 本身不合法，落地前可以先用 `yaml.safe_load` 自查 - 训练子进程失败：优先查看 `train_output_training_log_path`，常见原因包括模型路径错误、`llamafactory_dir` 不存在、CUDA OOM、未在 `dataset_info.json` 中注册数据集 -- 长时间训练：当前 `training_execution_node` 没有固定等待上限，调用方需要保持运行；如果调用被中断，Trainer 会取消训练进程组 -- SwanLab 日志路径为空：检查模板中的 `report_to` 是否被覆盖成 `none`，以及 `swanlab_api_key` 是否生效 +- 长时间训练：当前 `training_execution_node` 没有固定等待上限；调用方正常存活时会同步等待。若调用方意外中断，持久化 Trainer Worker 会继续训练和收尾，可通过同一 `task_id`、`version_id` 及运行目录中的 `run_state.json` / `worker_result.pkl` 重新读取状态 +- 曲线或指标为空：检查 SFT 运行目录中的 `trainer_log.jsonl`、`metrics/metrics.json` 是否持续更新，并优先查看原始训练日志排查解析错误 ## 进阶：单独复用配置生成能力 @@ -318,8 +312,6 @@ config = gen.generate_config( model_name="/path/to/Qwen2.5-Coder-7B", output_dir="./output/sql_sft", template_path="loopai/skills/Trainer/templates/qwen2_5_coder_bird_full_sft.yaml", - use_swanlab=True, - swanlab_project="sql_sft", ) gen.save_config_as_yaml(config, "./output/sql_sft/training_config.yaml") ``` @@ -327,5 +319,5 @@ gen.save_config_as_yaml(config, "./output/sql_sft/training_config.yaml") ## 使用时最该关注什么 - 训练前：必填字段是否齐全，数据检查是否通过 -- 训练中：日志是否持续更新，SwanLab 是否能看到 loss 曲线 +- 训练中：日志和本地指标文件是否持续更新，前端是否能读取 loss 曲线 - 训练后：`training_checkpoints` 是否非空，`train_output_training_report_path` 是否生成成功，新 checkpoint 是否能被下一轮 `Judger` 加载 diff --git a/ui/package.json b/ui/package.json index 75981761..86aef6bd 100644 --- a/ui/package.json +++ b/ui/package.json @@ -6,7 +6,7 @@ "type": "module", "scripts": { "dev": "vite", - "build": "node --max-old-space-size=4096 ./node_modules/vite/bin/vite.js build", + "build": "vite build", "preview": "vite preview", "lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs --fix --ignore-path .gitignore", "format": "prettier --write src/", diff --git a/ui/src/components/manage/mainFlow/nodes/agentNode/statusPreview/trainState/index.vue b/ui/src/components/manage/mainFlow/nodes/agentNode/statusPreview/trainState/index.vue index ce292a37..96ce03fb 100644 --- a/ui/src/components/manage/mainFlow/nodes/agentNode/statusPreview/trainState/index.vue +++ b/ui/src/components/manage/mainFlow/nodes/agentNode/statusPreview/trainState/index.vue @@ -68,14 +68,26 @@ export default { return this.statusList.map((item) => item.step) }, chartDataAttrs() { - let arr = [] - for (let key in this.statusList[0]) { - if (key === 'step') continue - if (typeof this.statusList[0][key] === 'number') { - arr.push(key) - } - } - return arr + if (!this.statusList.length) return [] + const keys = new Set() + this.statusList.forEach((item) => { + Object.keys(item).forEach((key) => { + if (key !== 'step' && typeof item[key] === 'number') keys.add(key) + }) + }) + const preferred = [ + 'loss', + 'eval_loss', + 'reward', + 'val_score', + 'policy_loss', + 'accuracy', + 'learning_rate' + ] + return [ + ...preferred.filter((key) => keys.has(key)), + ...Array.from(keys).filter((key) => !preferred.includes(key)) + ].slice(0, 8) } }, mounted() { @@ -130,7 +142,10 @@ export default { if (item.log_line) { this.loglines.push(item.log_line) } - if (item.loss != undefined && item.step != undefined) { + const hasNumericMetric = Object.keys(item).some( + (key) => key !== 'step' && typeof item[key] === 'number' + ) + if (item.step != undefined && hasNumericMetric) { this.statusList.push(item) } }) diff --git a/ui/src/components/manage/mainFlow/nodes/agentNode/statusPreview/trainState/lineChart.vue b/ui/src/components/manage/mainFlow/nodes/agentNode/statusPreview/trainState/lineChart.vue index 4222c3fb..e65dc0ee 100644 --- a/ui/src/components/manage/mainFlow/nodes/agentNode/statusPreview/trainState/lineChart.vue +++ b/ui/src/components/manage/mainFlow/nodes/agentNode/statusPreview/trainState/lineChart.vue @@ -62,7 +62,8 @@ onMounted(() => { data: props.data, borderColor: 'rgba(126, 113, 208, 1)', backgroundColor: 'rgba(126, 113, 208, 0.2)', - tension: 0.4 + tension: 0.4, + spanGaps: true } ] },