Source code for wbc_mjlab.rl.wm.config

"""Configuration of world-model training (``--algo wm``).

Defaults follow SuperMimic (Liao et al., arXiv:2610.28480, Table A.5).
"""

from __future__ import annotations

from dataclasses import dataclass, field, fields, replace
from typing import Literal

from mjlab.envs import ManagerBasedRlEnvCfg
from mjlab.rl import RslRlModelCfg, RslRlOnPolicyRunnerCfg

from wbc_mjlab.rl.policy import TrackingRunnerCfg, tracking_runner_cfg


[docs] @dataclass class WmAlgorithmCfg: """World model fit to simulator steps; actor trained by backpropagation through it.""" window: int = 8 """Steps of the autoregressive windows the world model is fit on (k).""" model_hidden_dims: tuple[int, ...] = field( default=(2048, 2048, 2048), metadata={"checkpoint": True} ) """Dynamics network.""" heads_hidden_dims: tuple[int, ...] = field( default=(512, 256), metadata={"checkpoint": True} ) """Step heads (termination, learned reward terms).""" dynamics_mode: Literal["increment", "integrated", "velocity"] = field( default="increment", metadata={"checkpoint": True} ) """Full increments, integrated pose residuals, or velocity-only integration.""" velocity_blend: float | None = field( default=1.0, metadata={"checkpoint": True, "nullable": True, "kind": "number"} ) """Integrated pose prior: new-velocity share; None derives (n+1)/(2n) from decimation.""" dynamics_loss_scale: Literal["output", "increment", "physical"] = field( default="output", metadata={"checkpoint": True} ) """Loss units: output statistics, full increment statistics, or fixed physical tolerances.""" physical_error_scales: tuple[float, ...] = field( default=(0.01, 0.02, 0.5, 1.0, 0.1, 3.14), metadata={"checkpoint": True} ) """Fixed tolerances: root m/rad/m/s/rad/s, joint rad/rad/s; physical loss only. Pose/joint tolerances follow official SuperMimic; explicit root velocity supervision is our extension. Scales never change with replay statistics. """ critic_inputs: Literal["actor", "structured"] = field( default="actor", metadata={"checkpoint": True} ) """Structured: physical state/tracking/context, clean goals and policy history.""" critic_layer_norm: bool = field(default=False, metadata={"checkpoint": True}) critic_activation: Literal["task", "elu", "gelu", "silu"] = field( default="task", metadata={"checkpoint": True} ) model_optimizer: Literal["adam", "muon"] = "adam" actor_optimizer: Literal["adam", "muon"] = "adam" muon_weight_decay: float = 0.0 """Decoupled weight decay of the Muon groups (world model and actor); PartyMimic: 0.1.""" actor_muon_io: bool = False """Muon also on the first and last Linear of the actor's MLPs (PartyMimic default).""" fused_optimizer: bool = False skip_nonfinite_updates: bool = False critic_norm_eps: float = 1e-4 """Minimum critic input scale; Blink uses 0.01 to limit near-constant-feature gain.""" bound_critic: bool = False """Cap bootstrap values at the task reward upper bound / (1-gamma).""" termination_mode: Literal["learned", "hybrid"] = "learned" """Hybrid uses task tracking thresholds as a lower bound on imagined termination.""" termination_margin_weight: float = 0.0 """Reward rate subtracted per tracking term at its failure threshold; zero disables.""" termination_margin_start: float = 0.5 """Fraction of the configured threshold where the margin cost starts, in [0, 1).""" policy_ema: float = 0.0 """Deployment policy exponential average; zero disables. Live weights resume training.""" mirror_probability: float = 0.0 """Share of dynamics windows reflected using robot symmetry; step heads remain unmirrored.""" actor_lr_multiplier: float = 1.0 actor_lr_decay_start: int = 1400 actor_lr_decay_updates: int = 300 actor_lr_return_threshold: float | None = field( default=None, metadata={"kind": "number", "nullable": True} ) """Start multiplier decay at this normalized simulator reward, or the update deadline.""" recent_failure_share: float | None = field( default=None, metadata={"kind": "number", "nullable": True} ) """Override the task's recent-failure reset mixture when set.""" probe_share: float = 0.15 collection_cuda_graphs: bool = False probe_cuda_graphs: bool = False """Capture policy inference; physics graphs remain owned by mjlab.""" probe_interval: int = 0 """Actor updates between curriculum probe batches; zero disables.""" probe_start: int = 1000 probe_num_envs: int = 128 probe_tile_s: float = 6.0 probe_lead_s: tuple[float, float] = (4.0, 9.0) probe_steps_per_iteration: int = 20 """Bounded probe work per runner iteration; zero completes a sweep synchronously.""" probe_async: bool = False """Run probe chunks on a CUDA side stream with explicit snapshot/event ownership.""" compile_collection: bool = False """Compile the collection actor and critic loss; simulation uses mjlab's own graphs.""" separate_action_noise: bool = False """Exploration only in execution; observations and action-rate reward use clean actions.""" self_imitation: float = 0.0 """Weight of self-imitation at imagination starts: regress the actor onto the recorded (exploring) action where the return the simulator recorded beats V(s₀), weighted by the advantage (PartyMimic ``--sil``: 1.0). Zero disables.""" self_imitation_max_weight: float = 5.0 """Cap of the advantage weight, in units of the batch's mean absolute advantage.""" value_gradient: float = 0.0 """Critic loss weight on matching ∇ₛV to the backpropagated imagined-return gradient at imagined states (PartyMimic ``--v_gcons``: 0.5). Zero disables.""" value_gradient_states: int = 4096 """Imagined states per critic minibatch in the value-gradient loss.""" value_gradient_epochs: int = 1 """Critic epochs whose minibatches include the value-gradient loss.""" bf16: bool = False """Run the networks' matmuls (world model, critic, policy head) in bfloat16 autocast; integration, kinematics, rewards and normalization stay float32 (PartyMimic ``--amp bf16``). CUDA only; exported policies stay float32.""" latent_aux: float = 0.0 """Weight of predicting the goal frames' joint positions from the actor's reference latent (``actor.latent_aux`` head; PartyMimic ``--kp_aux``: 1.0 with keypoint goals). Zero disables.""" scale_lr_by_world_size: bool = True """Distributed runs use sqrt(world_size) times the configured learning rate.""" diagnostic_interval: int = 100 """Updates between independent replay rollout diagnostics; zero disables them.""" diagnostic_batch: int = 256 model_batch: int = 2048 """Windows per world-model gradient step.""" model_steps: int = 10 """World-model gradient steps per update.""" horizon: int = 32 """Imagined steps per rollout (H); the critic bootstraps beyond.""" imagined_rollouts: int = 4096 """Rollouts per policy update, started from replay states.""" updates_per_iteration: int = 2 """Updates per runner iteration (``num_steps_per_env`` env steps): 2 per 5 steps is the paper's update-to-data ratio of 0.4.""" gamma: float = 0.99 lam: float = 0.95 """GAE λ of the critic targets.""" action_noise: float = 0.1 """Std of the Gaussian noise on actions in the simulator (fixed).""" learning_rate: float = 2.0e-4 """World model, actor and critic.""" lr_final_ratio: float = 0.03 """Learning rates decay (cosine) to this fraction...""" lr_decay_updates: int = 250_000 """...over this many updates, then stay constant.""" max_grad_norm: float = 1.0 actor_update: Literal["pathwise", "action_targets"] = field( default="pathwise", metadata={"checkpoint": True} ) """Direct return gradient, or regress onto gradient-improved actions (Nabla-inspired).""" action_target_steps: int = 8 action_target_windows: int = field(default=1, metadata={"checkpoint": True}) action_target_batch: int = 4096 action_target_delta: float = 0.03 """Initial RMS target displacement in joint-setpoint radians.""" action_target_delta_floor: float = 0.3 """Cosine annealing floor relative to delta, using lr_decay_updates.""" action_gradient_clip_ratio: float = 0.0 """Per-step action cotangent limit / median live norm (NablaMimic: 3); 0 disables.""" actor_grad_clip: float = 0.1 """Element-wise clip of actor gradients, after the norm clip.""" critic_epochs: int = 5 critic_mini_batches: int = 4 replay_steps: int = 1000 """Steps of every env kept in the replay.""" warmup_steps: int = 32 """Env steps collected before the first update.""" compile: bool = True """``torch.compile`` the imagination step and the world-model losses (first update compiles for a few minutes; off: slower, for debugging).""" cuda_graphs: bool = True """Run each world-model and policy update as one captured CUDA graph (thousands of small kernels launched at once); with ``compile`` only.""" class_name: str = "wbc_mjlab.rl.wm.algorithm:WorldModelRL"
@dataclass class WmRunnerCfg(TrackingRunnerCfg): """Runner settings for ``--algo wm``: the PPO runner's, with :class:`WmAlgorithmCfg`. ``actor`` is the policy (exploration noise: ``algorithm.action_noise``); ``critic`` sizes the value network, which sees the world-model state, the tracking features and the domain-randomization context instead of an observation group. """ algorithm: WmAlgorithmCfg = field(default_factory=WmAlgorithmCfg) # type: ignore[assignment] def wm_runner_cfg(ppo: RslRlOnPolicyRunnerCfg) -> WmRunnerCfg: """The ``wm`` runner config of a robot, from its PPO one.""" # The actor's noise is set by ``algorithm.action_noise`` (fixed, not learned). actor = replace( tracking_runner_cfg(ppo).actor, obs_normalization=True, distribution_cfg=None ) return WmRunnerCfg( seed=ppo.seed, num_steps_per_env=5, max_iterations=160_000, obs_groups=dict(ppo.obs_groups), save_interval=1000, experiment_name=ppo.experiment_name, logger=ppo.logger, wandb_project=ppo.wandb_project, clip_actions=ppo.clip_actions, upload_model=ppo.upload_model, actor=actor, critic=RslRlModelCfg(hidden_dims=(256, 128)), algorithm=WmAlgorithmCfg(), )
[docs] def wm_env_cfg(cfg: ManagerBasedRlEnvCfg) -> ManagerBasedRlEnvCfg: """Remove unobserved ongoing forces; offer optional, explicitly recorded velocity kicks. Kicks preserve a pre-disturbance target and cut model windows (perturbations.py). """ removed = [ name for name in ("push_robot", "pull_robot") if cfg.events.pop(name, None) ] motion = cfg.commands.get("motion") if getattr(motion, "assistive_wrench_enabled", False): motion.assistive_wrench_enabled = False removed.append("assistive wrench") if "assistive_wrench" in cfg.events: cfg.events["assistive_wrench"].params["enabled"] = False if removed: print(f"[INFO] wm: off for world-model training: {', '.join(removed)}") # Constraint capacity is a method-wide safety margin, including Lookahead + wm-blink. cfg.sim.njmax = max(cfg.sim.njmax or 0, 512) cfg.sim.contact_sensor_maxmatch = max(cfg.sim.contact_sensor_maxmatch or 0, 128) from mjlab.managers.event_manager import EventTermCfg from wbc_mjlab.rl.wm.perturbations import EVENT, inter_step_velocity_kick cfg.events.setdefault( EVENT, EventTermCfg( func=inter_step_velocity_kick, mode="step", params={ "enabled": False, "interval_range_s": (5.0, 10.0), "velocity_ranges": ( (-0.5, 0.5), (-0.5, 0.5), (0.0, 0.0), (-0.25, 0.25), (-0.25, 0.25), (0.0, 0.0), ), "initial_scale": 0.2, "ramp_duration_s": 60.0, "clean_episode_fraction": 0.2, }, ), ) # Pre-reset capture retains the state reached by a terminal action; see recording.py # for SuperMimic / SuperMimic-Reproduced provenance. from mjlab.envs.mdp.actions import JointPositionActionCfg from mjlab.managers.recorder_manager import RecorderTermCfg from wbc_mjlab.env.mdp.actions import TrackingJointPositionActionCfg from wbc_mjlab.rl.wm.recording import RECORDER, TerminalStateRecorder for name, action in cfg.actions.items(): if type(action) is JointPositionActionCfg: cfg.actions[name] = TrackingJointPositionActionCfg( **{f.name: getattr(action, f.name) for f in fields(action) if f.init} ) cfg.recorders[RECORDER] = RecorderTermCfg( func=TerminalStateRecorder, params={"command_name": "motion"} ) return cfg