"""Training recipes: how a preset learns — the method (PPO or the world model) and its
settings — saved by name, independent of any preset.
A preset is the task (what the policy sees, how it is rewarded, where episodes start); a recipe
is how it is trained. Train picks one of each; world-model recipes share the
trainer's observation, action and environment support constraints.
An unedited fresh job passes ``--recipe <name>`` to resolve defaults in the worker;
custom edits use ``--agent.<key> <value>``. Resolved configs are saved in the run
for reproducibility and resume.
Built-in recipes: ``ppo`` and ``wm`` (task defaults), ``wm-24gb`` (2048 imagined
rollouts), ``wm-encoder``, ``wm-blink`` and ``wm-party`` (optional experiments). Yours live in
``<config dir>/recipes/<name>.yaml`` (:func:`wbc_mjlab.studio.presets.config_dir`).
"""
from __future__ import annotations
import time
from dataclasses import fields as dataclass_fields
from pathlib import Path
from typing import Any
import yaml
from wbc_mjlab.rl.algos import ALGORITHMS, Algo, check_algo, rl_cfg
from wbc_mjlab.rl.policy import OLD_ACTOR_KEYS, PolicyCfg, migrate_actor_settings
from wbc_mjlab.rl.wm.config import WmAlgorithmCfg
# Runner config keys a recipe can set (prefixes; the rest is wiring).
_KEEP = (
"seed",
"num_steps_per_env",
"save_interval",
"light_checkpoints",
"clip_actions",
"algorithm",
"actor.hidden_dims",
"actor.activation",
"critic.hidden_dims",
) + tuple(
"actor." + f.name for f in dataclass_fields(PolicyCfg) if f.metadata.get("setting")
)
_SKIP = {"algorithm.class_name", "algorithm.optimizer", "algorithm.share_cnn_encoders"}
ARCHITECTURE = (
(
"actor.hidden_dims",
"actor.activation",
"critic.hidden_dims",
)
+ tuple(
"actor." + f.name
for f in dataclass_fields(PolicyCfg)
if f.metadata.get("checkpoint")
)
+ tuple(
"algorithm." + f.name
for f in dataclass_fields(WmAlgorithmCfg)
if f.metadata.get("checkpoint")
)
)
"""Settings a checkpoint fixes: continuing a run keeps that run's values."""
SECTIONS: dict[Algo, tuple[tuple[str, str], ...]] = {
"ppo": (("algorithm.", "Optimisation"),),
"wm": (("algorithm.", "World model & imagination"),),
}
_COMMON_SECTIONS = (
("actor.", "Policy network"),
("critic.", "Value network"),
("", "Rollouts"),
)
HELP = {
"algorithm.actor_update": "direct rollout gradient or regression onto improved action targets",
"algorithm.action_gradient_clip_ratio": "per-sample action gradient limit relative to the live median; 0 disables",
"algorithm.action_target_delta": "RMS target step in joint-setpoint radians, before action bounds",
"algorithm.action_target_windows": "recent target windows to retain and save in full checkpoints",
"algorithm.termination_margin_weight": "reward rate subtracted at a tracking failure threshold; 0 disables",
"algorithm.termination_margin_start": "fraction of each tracking failure threshold where the penalty starts",
"algorithm.learning_rate": "base learning rate (actor multiplier configured separately)",
"algorithm.entropy_coef": "exploration bonus",
"actor.hidden_dims": "the policy both methods train and export; checkpoints of other sizes won't load",
"critic.hidden_dims": "value network",
"algorithm.imagined_rollouts": "rollouts per policy update; 2048 fits a 24 GB GPU",
"algorithm.horizon": "imagined steps per rollout; the critic bootstraps beyond",
"algorithm.window": "steps of the windows the world model is fit on",
"algorithm.model_hidden_dims": "dynamics network; smaller is faster, less accurate",
"algorithm.updates_per_iteration": "updates per iteration of num_steps_per_env env steps",
"algorithm.action_noise": "std of the exploration noise on actions in the simulator",
"algorithm.replay_steps": "steps of every env kept in the replay",
"algorithm.compile": "torch.compile the update (first update compiles for minutes)",
"algorithm.cuda_graphs": "run each update as one CUDA graph (with compile)",
"algorithm.dynamics_mode": "full state increments, integrated pose residuals, or velocity-only integration",
"algorithm.separate_action_noise": "keep clean actions in history and smoothing reward; noise only at execution",
"algorithm.diagnostic_interval": "updates between independent replay diagnostics; 0 disables",
"algorithm.muon_weight_decay": "decoupled weight decay of Muon matrices (PartyMimic: 0.1)",
"algorithm.actor_muon_io": "Muon on the policy's first and last layers too",
"algorithm.self_imitation": "imitate recorded actions whose simulated return beat the critic; 0 disables",
"algorithm.value_gradient": "fit the critic's state gradient to the imagined return's; 0 disables",
"algorithm.bf16": "run the network matmuls in bfloat16 (CUDA); integration and rewards stay float32",
"algorithm.latent_aux": "predict the goal frames' joints from the reference latent (needs actor.latent_aux); 0 disables",
"num_steps_per_env": "env steps of every env per iteration",
}
_OFFICIAL_OPTIONS = {
"algorithm.dynamics_mode": "velocity",
"algorithm.velocity_blend": None,
"algorithm.dynamics_loss_scale": "physical",
"algorithm.critic_inputs": "structured",
"algorithm.critic_layer_norm": True,
"algorithm.critic_activation": "gelu",
"algorithm.critic_norm_eps": 0.01,
"algorithm.bound_critic": True,
"algorithm.termination_mode": "hybrid",
"algorithm.skip_nonfinite_updates": True,
"algorithm.model_optimizer": "muon",
"algorithm.actor_optimizer": "muon",
"algorithm.fused_optimizer": True,
"actor.architecture": "motion_encoder",
"actor.reference_residual": False,
"actor.hidden_dims": [2048, 1024, 512, 256],
"actor.activation": "gelu",
"algorithm.imagined_rollouts": 2048,
"clip_actions": 20.0,
"save_interval": 250,
"light_checkpoints": False,
}
BUILTIN: dict[str, dict] = {
"wm-nabla": {
"algo": "wm",
"label": "World model · Nabla-inspired",
"description": "Experimental action-gradient clipping and action-target regression through the learned world model. No JAX or differentiable simulator; not a NablaMimic reproduction.",
"settings": {
"algorithm.actor_update": "action_targets",
"algorithm.action_gradient_clip_ratio": 3.0,
"algorithm.action_target_steps": 8,
"algorithm.action_target_windows": 1,
"algorithm.action_target_batch": 4096,
"algorithm.action_target_delta": 0.03,
"algorithm.action_target_delta_floor": 0.3,
"algorithm.imagined_rollouts": 1024,
"algorithm.skip_nonfinite_updates": True,
"algorithm.critic_norm_eps": 0.01,
"algorithm.bound_critic": True,
"algorithm.termination_mode": "hybrid",
"clip_actions": 20.0,
"light_checkpoints": False,
},
},
"wm-supermimic": {
"algo": "wm",
"label": "World model · official-inspired",
"description": "Experimental velocity-only dynamics, fixed physical loss and structured critic. Shares the existing trainer; not a paper reproduction.",
"settings": dict(_OFFICIAL_OPTIONS),
},
"wm-blink": {
"algo": "wm",
"label": "World model · BlinkMimic",
"description": "Experimental BlinkMimic learner for motion presets. Optional joint-reference shortcut and symmetry augmentation; use Wbc-G1-Blink for contact shaping. Single GPU.",
"settings": {
"algorithm.model_hidden_dims": [1024, 1024, 1024],
"algorithm.model_steps": 5,
"algorithm.imagined_rollouts": 2048,
"algorithm.dynamics_mode": "integrated",
"algorithm.velocity_blend": None,
"algorithm.dynamics_loss_scale": "increment",
"algorithm.model_optimizer": "muon",
"algorithm.actor_optimizer": "muon",
"algorithm.fused_optimizer": True,
"algorithm.skip_nonfinite_updates": True,
"algorithm.policy_ema": 0.995,
"algorithm.critic_norm_eps": 0.01,
"algorithm.bound_critic": True,
"algorithm.termination_mode": "hybrid",
"clip_actions": 20.0,
"algorithm.mirror_probability": 0.0,
"algorithm.learning_rate": 4e-4,
"algorithm.actor_lr_multiplier": 4.0,
"algorithm.actor_lr_decay_start": 1400,
"algorithm.actor_lr_decay_updates": 300,
"algorithm.actor_lr_return_threshold": 0.75,
"algorithm.recent_failure_share": 0.3,
"algorithm.probe_interval": 125,
"algorithm.probe_start": 1000,
"algorithm.probe_share": 0.15,
"algorithm.compile_collection": True,
"algorithm.collection_cuda_graphs": True,
"algorithm.probe_cuda_graphs": True,
"actor.architecture": "motion_encoder",
# Task-dependent features remain explicit opt-ins: EE observations need no
# joint reference, and custom robots need not define a reflection mapping.
"actor.reference_residual": False,
"actor.hidden_dims": [2048, 1024, 512, 256],
"actor.activation": "silu",
"save_interval": 250,
"light_checkpoints": False,
},
},
"wm-party": {
"algo": "wm",
"label": "World model · PartyMimic",
"description": "Experimental BlinkMimic learner with PartyMimic's additions: Muon weight decay on every policy matrix, self-imitation and value-gradient consistency; no curriculum probes. Use Wbc-G1-Party for its narrower pose kernels and diversity-balanced resets. Single GPU.",
"settings": {}, # filled below from wm-blink
},
"ppo": {
"algo": "ppo",
"label": "PPO",
"description": "PPO with the preset's settings",
"settings": {},
},
"wm": {
"algo": "wm",
"label": "World model · default",
"description": "SuperMimic's settings: 4096 imagined rollouts (needs > 24 GB)",
"settings": {},
},
"wm-24gb": {
"algo": "wm",
"label": "World model · 24 GB GPU",
"description": "2048 imagined rollouts: fits a 24 GB GPU (RTX 4090)",
"settings": {"algorithm.imagined_rollouts": 2048},
},
"wm-encoder": {
"algo": "wm",
"label": "World model · combined experiment",
"description": "Optional SuperMimic encoder/decoder and integrated residual experiment",
"settings": {
"actor.architecture": "motion_encoder",
"algorithm.dynamics_mode": "integrated",
"algorithm.separate_action_noise": True,
"algorithm.imagined_rollouts": 2048,
"actor.hidden_dims": [2048, 1024, 512, 256],
"actor.activation": "silu",
},
},
}
# PartyMimic (Roboparty, 2026) train.py defaults, MIT; see THIRD_PARTY_NOTICES. Its
# reference residual and symmetry augmentation stay opt-ins here, as in wm-blink.
BUILTIN["wm-party"]["settings"] = {
**{k: v for k, v in BUILTIN["wm-blink"]["settings"].items() if "probe" not in k},
"algorithm.probe_interval": 0,
"algorithm.warmup_steps": 64,
"algorithm.muon_weight_decay": 0.1,
"algorithm.actor_muon_io": True,
"algorithm.self_imitation": 1.0,
"algorithm.value_gradient": 0.5,
"algorithm.bf16": True,
}
def recipes_dir() -> Path:
from wbc_mjlab.studio.presets import config_dir
return config_dir() / "recipes"
def _path(name: str) -> Path:
from wbc_mjlab.studio.presets import _check
return recipes_dir() / f"{_check(name)}.yaml"
# ---------------------------------------------------------------------- settings
[docs]
def fields(task: str, algo: Algo) -> list[dict]:
"""Every setting of ``algo`` for ``task``: ``{key, kind, value, section, main, help}``
with the task's value as ``value`` (``main``: the method's settings shown first)."""
from wbc_mjlab.studio.presets import _walk_dataclass
check_algo(algo)
sections = (*SECTIONS[algo], *_COMMON_SECTIONS)
out = []
cfg = rl_cfg(task, algo)
for f in _walk_dataclass(cfg):
key = f["key"]
if key in _SKIP or not any(key == k or key.startswith(k + ".") for k in _KEEP):
continue
section = next(title for prefix, title in sections if key.startswith(prefix))
main = key in ALGORITHMS[algo].settings
if key == "actor.goal_terms":
from mjlab.tasks.registry import load_env_cfg
from wbc_mjlab.rl.policy import goal_term_names
terms = load_env_cfg(task).observations["actor"].terms
try:
inferred = ", ".join(goal_term_names(terms))
except ValueError:
inferred = "select explicit terms for this preset"
f["help"] += "; inferred: " + inferred
out.append(
f | {"section": section, "main": main, "help": HELP.get(key, f.get("help", ""))}
)
order = [title for _, title in sections]
return sorted(out, key=lambda f: order.index(f["section"]))
def check_settings(task: str, algo: Algo, settings: dict | None) -> dict:
"""``settings`` validated against ``algo``'s settings for ``task`` (unknown keys raise)."""
settings = migrate_recipe_settings(settings or {})
known = {f["key"] for f in fields(task, algo)}
for key in settings or {}:
if key not in known:
raise ValueError(f"{key} is not a setting of {ALGORITHMS[algo].label}")
actor = rl_cfg(task, algo).actor
architecture = settings.get("actor.architecture", actor.architecture)
if architecture in ("motion_encoder", "xattn"):
from mjlab.tasks.registry import load_env_cfg
from wbc_mjlab.rl.policy import goal_term_names
goal_term_names(
load_env_cfg(task).observations["actor"].terms,
settings.get("actor.goal_terms", actor.goal_terms),
)
if settings.get("actor.reference_residual", actor.reference_residual):
from mjlab.tasks.registry import load_env_cfg
from wbc_mjlab.env.mdp.actions import ReferenceJointPositionActionCfg
from wbc_mjlab.env.mdp.observations import ref_joint_pos
env = load_env_cfg(task)
terms = [
t
for t in env.observations["actor"].terms.values()
if t is not None and t.func is ref_joint_pos
]
if (
len(terms) != 1
or terms[0].params.get("frames", 1) < 2
or isinstance(env.actions.get("joint_pos"), ReferenceJointPositionActionCfg)
):
raise ValueError(
"reference residual policy requires joint lookahead and default-relative actions"
)
return dict(settings or {})
def migrate_recipe_settings(settings):
aliases = {"algorithm." + old: "actor." + new for old, new in OLD_ACTOR_KEYS.items()}
out = {key: value for key, value in settings.items() if key not in aliases}
for old, new in aliases.items():
if old in settings:
out.setdefault(new, settings[old])
return out
[docs]
def architecture_of(run_dir: Path | str) -> dict:
"""The :data:`ARCHITECTURE` settings run ``run_dir`` was trained with (those its
``params/agent.yaml`` holds)."""
from wbc_mjlab.run_params import read_run_params
agent = migrate_actor_settings(read_run_params(run_dir, "agent"))
out = {}
for key in ARCHITECTURE:
value: Any = agent
for part in key.split("."):
value = value.get(part) if isinstance(value, dict) else None
if value is not None:
out[key] = list(value) if isinstance(value, (list, tuple)) else value
return out
# ---------------------------------------------------------------------- storage
[docs]
def list_recipes() -> list[dict]:
"""Built-in recipes, then yours: ``{name, algo, label, description, settings, builtin}``."""
out = [r | {"name": n, "builtin": True} for n, r in BUILTIN.items()]
d = recipes_dir()
for p in sorted(d.glob("*.yaml")) if d.is_dir() else []:
try:
out.append(load(p.stem))
except (OSError, ValueError, yaml.YAMLError):
continue
return out
def load(name: str) -> dict:
if name in BUILTIN:
return BUILTIN[name] | {"name": name, "builtin": True}
data = yaml.safe_load(_path(name).read_text()) or {}
check_algo(data.get("algo", ""))
return {
"name": name,
"algo": data["algo"],
"label": name,
"description": data.get("description", ""),
"settings": migrate_recipe_settings(data.get("settings") or {}),
"builtin": False,
"updated": data.get("updated", ""),
}
[docs]
def save(name: str, algo: str, settings: dict, description: str = "") -> dict:
if name in BUILTIN:
raise ValueError(f"{name!r} is a built-in recipe: pick another name")
check_algo(algo)
data = {
"algo": algo,
"description": description,
"updated": time.strftime("%Y-%m-%d %H:%M"),
"settings": {k: _plain(v) for k, v in (settings or {}).items()},
}
p = _path(name)
p.parent.mkdir(parents=True, exist_ok=True)
p.write_text(yaml.safe_dump(data, sort_keys=False))
return load(name)
def delete(name: str) -> None:
if name in BUILTIN:
raise ValueError("built-in recipes can't be deleted")
_path(name).unlink(missing_ok=True)
def default(algo: Algo) -> str:
"""The recipe Train starts ``algo`` with: on a GPU under 40 GB the world model's 24 GB one
(4096 imagined rollouts ran out of memory on 24 GB)."""
if algo == "wm":
import torch
if torch.cuda.is_available():
gb = torch.cuda.get_device_properties(0).total_memory / 2**30
if gb < 40:
return "wm-24gb"
return algo
def _plain(v: Any) -> Any:
return list(v) if isinstance(v, tuple) else v
__all__ = [
"ARCHITECTURE",
"BUILTIN",
"architecture_of",
"check_settings",
"default",
"delete",
"fields",
"list_recipes",
"load",
"recipes_dir",
"save",
]