"""Training algorithms of a run: ``ppo`` (default) or ``wm`` (world model, :mod:`wbc_mjlab.rl.wm`).
Every algorithm trains the same tasks and presets (env, motion options) and exports the same
policy; it brings its own runner config (``params/agent.yaml``, which Play reads back) and
may adapt the env. :data:`ALGORITHMS` describes them for the CLI and the Studio (whose
recipes, :mod:`wbc_mjlab.studio.recipes`, are an algorithm and its settings).
"""
from __future__ import annotations
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Literal
Algo = Literal["ppo", "wm"]
ALGOS: tuple[Algo, ...] = ("ppo", "wm")
[docs]
@dataclass(frozen=True)
class AlgoInfo:
"""What an algorithm is, for listings and the Studio's Train form."""
id: Algo
label: str
summary: str
settings: tuple[str, ...] = ()
"""Runner config keys worth setting per run (``algorithm.imagined_rollouts`` …)."""
iterations: tuple[int, ...] = (1000, 5000, 30000, 100000)
"""Typical ``max_iterations`` budgets (an iteration's length differs per algorithm)."""
notes: str = ""
ALGORITHMS: dict[Algo, AlgoInfo] = {
"ppo": AlgoInfo(
id="ppo",
settings=("actor.architecture",),
label="PPO",
summary=(
"On-policy RL in the simulator: robust, works with every preset. "
"An iteration is num_steps_per_env (24) env steps of every env and one PPO update."
),
),
"wm": AlgoInfo(
id="wm",
label="World model",
summary=(
"SuperMimic: a world model learns the robot's dynamics from the simulator and the "
"policy is trained by backpropagation through imagined rollouts. "
"An iteration is 5 env steps and 2 updates."
),
settings=(
"actor.architecture",
"algorithm.imagined_rollouts",
"algorithm.horizon",
"algorithm.model_hidden_dims",
"algorithm.compile",
),
iterations=(5000, 20000, 60000, 160000),
notes=(
"Pushes, pulls and the assistive wrench are off (a deterministic model cannot "
"predict them). Optional recorded velocity kicks train recovery. "
"The first update compiles for a few minutes."
),
),
}
"""Every training algorithm (what ``--algo`` accepts), by id."""
def check_algo(algo: str) -> Algo:
if algo not in ALGORITHMS:
raise ValueError(f"unknown algorithm {algo!r}; choose from {', '.join(ALGOS)}")
return algo # type: ignore[return-value]
[docs]
def runner_cfg(ppo_cfg, algo: Algo):
"""``algo``'s runner config derived from a task's PPO runner config ``ppo_cfg`` (the
policy network, experiment and logger carry over)."""
check_algo(algo)
if algo == "wm":
from wbc_mjlab.rl.wm.config import wm_runner_cfg
return wm_runner_cfg(ppo_cfg)
from wbc_mjlab.rl.policy import tracking_runner_cfg
return tracking_runner_cfg(ppo_cfg)
[docs]
def rl_cfg(task_id: str, algo: Algo = "ppo"):
"""Runner config of ``task_id`` for ``algo``."""
from mjlab.tasks.registry import load_rl_cfg
return runner_cfg(load_rl_cfg(task_id), algo)
[docs]
def env_cfg(cfg, algo: Algo):
"""The env ``algo`` trains (and plays) ``cfg`` in: ``wm`` turns off external forces."""
check_algo(algo)
if algo == "wm":
from wbc_mjlab.rl.wm.config import wm_env_cfg
return wm_env_cfg(cfg)
return cfg
def select_algo(task_id: str, algo: Algo) -> None:
"""Make ``algo`` the task's runner (and env) config in this process, before mjlab's
train CLI reads them."""
from mjlab.tasks import registry
check_algo(algo)
if algo != "ppo":
entry = registry._REGISTRY[task_id] # mjlab has no public re-registration
registry._REGISTRY[task_id] = replace(
entry, env_cfg=env_cfg(entry.env_cfg, algo), rl_cfg=rl_cfg(task_id, algo)
)
[docs]
def run_algo(run_dir: Path | str) -> Algo:
"""Algorithm a run was trained with (``params/agent.yaml``; PPO when absent)."""
from wbc_mjlab.rl.wm.config import WmAlgorithmCfg
from wbc_mjlab.run_params import read_run_params
algorithm = read_run_params(run_dir, "agent").get("algorithm") or {}
return "wm" if algorithm.get("class_name") == WmAlgorithmCfg.class_name else "ppo"
[docs]
def run_rl_cfg(task_id: str, run_dir: Path | str):
"""Rebuild a run's architecture for Play/evaluation, including optional encoders.
Apply known dataclass fields only: YAML is data, never imported or evaluated.
"""
from wbc_mjlab.rl.policy import migrate_actor_settings
from wbc_mjlab.run_params import read_run_params, restore_config
cfg = rl_cfg(task_id, run_algo(run_dir))
restore_config(cfg, migrate_actor_settings(read_run_params(run_dir, "agent")))
return cfg
[docs]
def check_env(task_id: str, algo: Algo, motion_file: str, *, num_envs: int = 2) -> dict:
"""Build ``task_id``'s env for ``algo`` (``num_envs`` envs on ``motion_file``) and check
that ``algo`` can train it: ``{"ok", "error", "details"}``. For ``wm``, ``details`` names
the rewards computed exactly in imagination, those learned, and the randomization the
model is given as context."""
import torch
from mjlab.envs import ManagerBasedRlEnv
from mjlab.tasks.registry import load_env_cfg
check_algo(algo)
cfg = env_cfg(load_env_cfg(task_id), algo)
cfg.scene.num_envs = num_envs
cfg.commands["motion"].motion_file = motion_file
device = "cuda:0" if torch.cuda.is_available() else "cpu"
try:
env = ManagerBasedRlEnv(cfg=cfg, device=device)
except Exception as e: # noqa: BLE001
return {"ok": False, "error": f"env: {e}", "details": {}}
try:
if algo != "wm":
return {"ok": True, "error": "", "details": {}}
from wbc_mjlab.rl.wm.imagined import Imagination
env.reset()
try:
im = Imagination(env)
except ValueError as e:
return {"ok": False, "error": str(e), "details": {}}
return {
"ok": True,
"error": "",
"details": {
"exact_rewards": [n for n, _ in im.analytic_rewards],
"learned_rewards": [n for _, n, _ in im.learned_rewards],
"context": list(im.context_names),
},
}
finally:
env.close()
__all__ = [
"ALGORITHMS",
"ALGOS",
"Algo",
"AlgoInfo",
"check_algo",
"check_env",
"env_cfg",
"rl_cfg",
"run_algo",
"run_rl_cfg",
"runner_cfg",
"select_algo",
]