32a241c28f
Some checks failed
nightly / Test against latest dependencies (py3.10) (push) Has been cancelled
nightly / Test against latest dependencies (py3.13) (push) Has been cancelled
tests / tests (3.13, locked) (push) Has been cancelled
tests / tests (3.13, unlocked) (push) Has been cancelled
tests / pyright (3.10) (push) Has been cancelled
tests / lint-format (push) Has been cancelled
tests / tests (3.10, locked) (push) Has been cancelled
tests / tests (3.11, locked) (push) Has been cancelled
tests / tests (3.12, locked) (push) Has been cancelled
tests / pyright (3.11) (push) Has been cancelled
tests / pyright (3.12) (push) Has been cancelled
tests / pyright (3.13) (push) Has been cancelled
tests / ty-check (3.10) (push) Has been cancelled
tests / ty-check (3.11) (push) Has been cancelled
tests / ty-check (3.12) (push) Has been cancelled
tests / ty-check (3.13) (push) Has been cancelled
tests / stubs (push) Has been cancelled
tests / smoke-test (push) Has been cancelled
Docker / check_paths (push) Has been cancelled
docs / build (push) Has been cancelled
Docker / build (push) Has been cancelled
Upstream: https://github.com/michaelgillett/mjlab Upstream-Commit: c19f713c415a699a79d71cd96aa13c3104a05047 Upstream-Branch: main
485 lines
18 KiB
ReStructuredText
485 lines
18 KiB
ReStructuredText
.. _environment_config:
|
|
|
|
Environment Configuration
|
|
=========================
|
|
|
|
A single ``ManagerBasedRlEnvCfg`` dataclass fully specifies an mjlab
|
|
environment: the physical world, the agent's interface to it, and the
|
|
MDP defined on top. Because everything lives in one flat
|
|
object, an environment can be inspected, copied, and modified without
|
|
navigating a class hierarchy.
|
|
|
|
For a broad orientation to mjlab before reading this page, start with
|
|
:ref:`architecture_overview`.
|
|
|
|
|
|
.. _env-config-skeleton:
|
|
|
|
Annotated skeleton
|
|
------------------
|
|
|
|
The complete set of fields on ``ManagerBasedRlEnvCfg`` is shown below with
|
|
inline comments. The fields marked with ``...`` must be provided; all others
|
|
have defaults.
|
|
|
|
.. code-block:: python
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
from mjlab.envs import ManagerBasedRlEnvCfg
|
|
from mjlab.managers.action_manager import ActionTermCfg
|
|
from mjlab.managers.command_manager import CommandTermCfg
|
|
from mjlab.managers.curriculum_manager import CurriculumTermCfg
|
|
from mjlab.managers.event_manager import EventTermCfg
|
|
from mjlab.managers.metrics_manager import MetricsTermCfg
|
|
from mjlab.managers.observation_manager import ObservationGroupCfg
|
|
from mjlab.managers.reward_manager import RewardTermCfg
|
|
from mjlab.managers.termination_manager import TerminationTermCfg
|
|
from mjlab.scene.scene import SceneCfg
|
|
from mjlab.sim.sim import SimulationCfg
|
|
from mjlab.viewer.viewer_config import ViewerConfig
|
|
|
|
|
|
@dataclass
|
|
class MyEnvCfg(ManagerBasedRlEnvCfg):
|
|
|
|
# --- Physics ---
|
|
|
|
decimation: int = 4
|
|
# Number of physics steps per policy step.
|
|
# Environment step duration = sim.mujoco.timestep * decimation.
|
|
|
|
sim: SimulationCfg = field(default_factory=SimulationCfg)
|
|
# Physics parameters: timestep, integrator, solver, contact settings.
|
|
# Default timestep is 0.002 s (500 Hz). Override with MujocoCfg.
|
|
|
|
scene: SceneCfg = ...
|
|
# Terrain, entities, and sensors. Also sets num_envs.
|
|
# Required; there is no default.
|
|
|
|
# --- Episode ---
|
|
|
|
episode_length_s: float = 20.0
|
|
# Episode duration in seconds.
|
|
# Steps = ceil(episode_length_s / (sim.mujoco.timestep * decimation)).
|
|
|
|
is_finite_horizon: bool = False
|
|
# False (default): time limit is an artificial cutoff. The agent
|
|
# receives a truncated signal and bootstraps value beyond the limit.
|
|
# True: time limit defines the task boundary. The agent receives a
|
|
# terminal done signal with no future value beyond it.
|
|
|
|
scale_rewards_by_dt: bool = True
|
|
# When True (default), each reward term is multiplied by step_dt so
|
|
# that cumulative episodic sums are invariant to simulation frequency.
|
|
# Set to False for algorithms that expect unscaled reward signals.
|
|
|
|
# --- Managers ---
|
|
|
|
observations: dict[str, ObservationGroupCfg] = field(default_factory=dict)
|
|
# Observation groups. Each key is a group name (e.g. "actor", "critic").
|
|
# Groups can differ in noise, history, delay, and concatenation.
|
|
|
|
actions: dict[str, ActionTermCfg] = field(default_factory=dict)
|
|
# Action terms. Each term controls one slice of the policy output
|
|
# and routes it to a specific entity's actuators.
|
|
|
|
rewards: dict[str, RewardTermCfg] = field(default_factory=dict)
|
|
# Reward terms. The manager computes a weighted sum each step.
|
|
|
|
terminations: dict[str, TerminationTermCfg] = field(default_factory=dict)
|
|
# Termination conditions. If empty, episodes never terminate early.
|
|
# Add a time_out term to enforce the episode length limit.
|
|
|
|
events: dict[str, EventTermCfg] = field(
|
|
default_factory=lambda: {
|
|
"reset_scene_to_default": EventTermCfg(
|
|
func=reset_scene_to_default,
|
|
mode="reset",
|
|
)
|
|
}
|
|
)
|
|
# Event terms for domain randomization and state resets.
|
|
# The default includes reset_scene_to_default, which resets all
|
|
# entities to their initial pose each episode. Override this dict
|
|
# to replace or extend the default reset behavior.
|
|
|
|
commands: dict[str, CommandTermCfg] = field(default_factory=dict)
|
|
# Command generators (e.g. velocity targets for locomotion).
|
|
# Commands are resampled at configurable intervals and on reset.
|
|
|
|
curriculum: dict[str, CurriculumTermCfg] = field(default_factory=dict)
|
|
# Curriculum terms that adjust training conditions based on performance.
|
|
|
|
metrics: dict[str, MetricsTermCfg] = field(default_factory=dict)
|
|
# Custom metrics logged as episode averages alongside reward terms.
|
|
|
|
# --- Misc ---
|
|
|
|
seed: int | None = None
|
|
# Random seed for reproducibility. If None, a random seed is chosen
|
|
# and stored back into this field after initialization.
|
|
|
|
viewer: ViewerConfig = field(default_factory=ViewerConfig)
|
|
# Camera position, resolution, and tracking target for rendering.
|
|
|
|
|
|
.. _env-config-term-pattern:
|
|
|
|
Term configuration pattern
|
|
--------------------------
|
|
|
|
All manager dictionaries follow the same pattern. Each entry maps a string
|
|
name to a term configuration object. The configuration always carries at
|
|
minimum a ``func`` field pointing to the callable that implements the term,
|
|
and a ``params`` dict of extra keyword arguments forwarded to that callable.
|
|
|
|
The manager calls ``func(env, **params)`` each step (or ``term(env, **params)``
|
|
when ``func`` is a class that has been instantiated). Term names are arbitrary;
|
|
they appear in training logs and are used only for identification.
|
|
|
|
.. rubric:: Reward terms
|
|
|
|
.. code-block:: python
|
|
|
|
from mjlab.envs import mdp
|
|
from mjlab.managers.reward_manager import RewardTermCfg
|
|
from mjlab.managers.scene_entity_config import SceneEntityCfg
|
|
|
|
rewards = {
|
|
"alive": RewardTermCfg(
|
|
func=mdp.is_alive,
|
|
weight=1.0,
|
|
),
|
|
"joint_torques": RewardTermCfg(
|
|
func=mdp.joint_torques_l2,
|
|
weight=-1e-4,
|
|
params={"asset_cfg": SceneEntityCfg("robot")},
|
|
),
|
|
"action_rate": RewardTermCfg(
|
|
func=mdp.action_rate_l2,
|
|
weight=-0.1,
|
|
),
|
|
}
|
|
|
|
``weight`` scales the function's output before it is summed into the total
|
|
reward. Negative weights produce penalties.
|
|
|
|
``params`` maps to keyword arguments of the function. For example,
|
|
``mdp.joint_torques_l2(env, asset_cfg=...)`` receives ``asset_cfg`` from the
|
|
``params`` dict. Any argument not listed in ``params`` must have a default
|
|
value in the function signature.
|
|
|
|
.. rubric:: Termination terms
|
|
|
|
.. code-block:: python
|
|
|
|
from mjlab.envs import mdp
|
|
from mjlab.managers.termination_manager import TerminationTermCfg
|
|
|
|
terminations = {
|
|
"time_out": TerminationTermCfg(
|
|
func=mdp.time_out,
|
|
time_out=True, # marks this as a truncation, not a failure
|
|
),
|
|
"fell_over": TerminationTermCfg(
|
|
func=mdp.bad_orientation,
|
|
params={"limit_angle": 1.22}, # ~70 degrees in radians
|
|
),
|
|
}
|
|
|
|
The ``time_out`` flag on ``TerminationTermCfg`` tells the manager to treat
|
|
this condition as a truncation rather than a terminal failure. Truncations
|
|
map to the ``truncated`` signal in the Gym interface; failures map to
|
|
``terminated``. This distinction matters for value bootstrapping in RL
|
|
algorithms.
|
|
|
|
.. rubric:: Event terms
|
|
|
|
.. code-block:: python
|
|
|
|
from mjlab.managers.event_manager import EventTermCfg
|
|
|
|
events = {
|
|
"reset_base": EventTermCfg(
|
|
func=mdp.reset_root_state_uniform,
|
|
mode="reset",
|
|
params={
|
|
"pose_range": {"yaw": (-3.14, 3.14)},
|
|
"velocity_range": {},
|
|
},
|
|
),
|
|
}
|
|
|
|
The ``mode`` field on ``EventTermCfg`` controls when the term fires:
|
|
at startup, on episode reset, or at regular intervals. See :ref:`events`
|
|
for the full treatment of lifecycle modes, built-in event functions, and
|
|
the relationship between events and domain randomization.
|
|
|
|
.. rubric:: Function-based vs. class-based terms
|
|
|
|
Terms can be plain functions or classes. Functions are suitable for stateless
|
|
computations; classes are useful when a term needs to cache expensive setup or
|
|
maintain state across steps.
|
|
|
|
A function-based term has the signature ``func(env, **params) -> Tensor``. A
|
|
class-based term is instantiated once with ``(cfg, env)`` and then called with
|
|
the same signature. Classes can optionally implement a ``reset(env_ids)`` hook
|
|
for per-episode state clearing.
|
|
|
|
.. code-block:: python
|
|
|
|
# Function-based (stateless)
|
|
RewardTermCfg(func=mdp.joint_torques_l2, weight=-0.01)
|
|
|
|
# Class-based (caches joint indices at init)
|
|
class MyReward:
|
|
def __init__(self, cfg, env):
|
|
self.joint_ids = resolve_joint_ids(cfg.params, env)
|
|
|
|
def __call__(self, env) -> torch.Tensor:
|
|
return compute_reward(env, self.joint_ids)
|
|
|
|
RewardTermCfg(func=MyReward, weight=1.0)
|
|
|
|
|
|
.. _env-config-timing:
|
|
|
|
Timing: decimation, timestep, and episode length
|
|
-------------------------------------------------
|
|
|
|
Three parameters jointly determine the temporal structure of the environment.
|
|
|
|
``sim.mujoco.timestep``
|
|
The physics integration step in seconds. The default is 0.002 s (500 Hz).
|
|
This is one of the most important parameters in any environment: smaller
|
|
values produce more stable physics but slow down simulation. See the MuJoCo
|
|
`performance tuning <https://mujoco.readthedocs.io/en/stable/modeling.html#performance-tuning>`_
|
|
guide for practical advice on choosing timesteps and solver settings.
|
|
|
|
``decimation``
|
|
The number of physics steps executed per policy step. The policy runs at
|
|
``1 / (timestep * decimation)`` Hz.
|
|
|
|
``episode_length_s``
|
|
The episode duration in seconds. The maximum number of policy steps per
|
|
episode is ``ceil(episode_length_s / (timestep * decimation))``.
|
|
|
|
**Concrete example.** The velocity task uses ``timestep=0.005`` (200 Hz
|
|
physics) and ``decimation=4``, giving a policy frequency of 50 Hz. With
|
|
``episode_length_s=20.0``, each episode runs for exactly 1000 policy steps.
|
|
|
|
.. code-block:: python
|
|
|
|
physics_dt = 0.005 # seconds per physics step (200 Hz)
|
|
decimation = 4 # physics steps per policy step
|
|
step_dt = 0.005 * 4 # = 0.02 s per policy step (50 Hz)
|
|
episode_len = 20.0 / 0.02 # = 1000 policy steps per episode
|
|
|
|
To read these values at runtime, use the environment properties:
|
|
|
|
.. code-block:: python
|
|
|
|
env.physics_dt # = cfg.sim.mujoco.timestep
|
|
env.step_dt # = cfg.sim.mujoco.timestep * cfg.decimation
|
|
env.max_episode_length # steps (int)
|
|
env.max_episode_length_s # seconds (float)
|
|
|
|
When ``scale_rewards_by_dt=True`` (the default), each reward term is
|
|
multiplied by ``step_dt`` before being returned. A reward function that
|
|
returns a constant value of 1.0 contributes ``step_dt`` per step and
|
|
approximately ``episode_length_s`` over a full episode, regardless of how
|
|
``decimation`` and ``timestep`` are set. Changing the simulation frequency
|
|
without disabling this scaling leaves reward magnitudes unchanged.
|
|
|
|
|
|
.. _env-config-subclassing:
|
|
|
|
Subclassing pattern
|
|
-------------------
|
|
|
|
mjlab uses plain dataclass inheritance rather than deeply nested class
|
|
hierarchies. To build a task-specific configuration, subclass
|
|
``ManagerBasedRlEnvCfg`` and override fields.
|
|
|
|
The recommended approach is to define the full configuration in a factory
|
|
function, then call it from robot-specific configs that override only the
|
|
fields that differ. The velocity task uses this pattern: ``make_velocity_env_cfg``
|
|
returns a fully assembled ``ManagerBasedRlEnvCfg``, and each robot
|
|
configuration calls the factory and patches in robot-specific values such
|
|
as the scene, joint name patterns, and action scale.
|
|
|
|
A condensed version of the factory illustrates the full assembly pattern:
|
|
|
|
.. code-block:: python
|
|
|
|
import math
|
|
from dataclasses import replace
|
|
|
|
from mjlab.envs import ManagerBasedRlEnvCfg
|
|
from mjlab.envs.mdp import dr
|
|
from mjlab.envs.mdp.actions import JointPositionActionCfg
|
|
from mjlab.managers.event_manager import EventTermCfg
|
|
from mjlab.managers.observation_manager import ObservationGroupCfg, ObservationTermCfg
|
|
from mjlab.managers.reward_manager import RewardTermCfg
|
|
from mjlab.managers.scene_entity_config import SceneEntityCfg
|
|
from mjlab.managers.termination_manager import TerminationTermCfg
|
|
from mjlab.scene import SceneCfg
|
|
from mjlab.sim import MujocoCfg, SimulationCfg
|
|
from mjlab.tasks.velocity import mdp
|
|
from mjlab.tasks.velocity.mdp import UniformVelocityCommandCfg
|
|
from mjlab.terrains import TerrainEntityCfg
|
|
from mjlab.terrains.config import ROUGH_TERRAINS_CFG
|
|
from mjlab.viewer import ViewerConfig
|
|
|
|
|
|
def make_velocity_env_cfg() -> ManagerBasedRlEnvCfg:
|
|
|
|
observations = {
|
|
"actor": ObservationGroupCfg(
|
|
terms={
|
|
"base_lin_vel": ObservationTermCfg(
|
|
func=mdp.builtin_sensor,
|
|
params={"sensor_name": "robot/imu_lin_vel"},
|
|
),
|
|
"joint_pos": ObservationTermCfg(func=mdp.joint_pos_rel),
|
|
"command": ObservationTermCfg(
|
|
func=mdp.generated_commands,
|
|
params={"command_name": "twist"},
|
|
),
|
|
# additional terms omitted for brevity
|
|
},
|
|
concatenate_terms=True,
|
|
enable_corruption=True,
|
|
),
|
|
"critic": ObservationGroupCfg(
|
|
terms={...},
|
|
concatenate_terms=True,
|
|
enable_corruption=False,
|
|
),
|
|
}
|
|
|
|
actions = {
|
|
"joint_pos": JointPositionActionCfg(
|
|
entity_name="robot",
|
|
actuator_names=(".*",),
|
|
scale=0.5,
|
|
use_default_offset=True,
|
|
)
|
|
}
|
|
|
|
commands = {
|
|
"twist": UniformVelocityCommandCfg(
|
|
entity_name="robot",
|
|
resampling_time_range=(3.0, 8.0),
|
|
ranges=UniformVelocityCommandCfg.Ranges(
|
|
lin_vel_x=(-1.0, 1.0),
|
|
lin_vel_y=(-1.0, 1.0),
|
|
ang_vel_z=(-0.5, 0.5),
|
|
heading=(-math.pi, math.pi),
|
|
),
|
|
)
|
|
}
|
|
|
|
events = {
|
|
"reset_base": EventTermCfg(
|
|
func=mdp.reset_root_state_uniform,
|
|
mode="reset",
|
|
params={
|
|
"pose_range": {"x": (-0.5, 0.5), "y": (-0.5, 0.5), "yaw": (-3.14, 3.14)},
|
|
"velocity_range": {},
|
|
},
|
|
),
|
|
"foot_friction": EventTermCfg(
|
|
mode="startup",
|
|
func=dr.geom_friction,
|
|
params={
|
|
"asset_cfg": SceneEntityCfg("robot", geom_names=[]),
|
|
"operation": "abs",
|
|
"ranges": (0.3, 1.2),
|
|
},
|
|
),
|
|
"push_robot": EventTermCfg(
|
|
func=mdp.push_by_setting_velocity,
|
|
mode="interval",
|
|
interval_range_s=(1.0, 3.0),
|
|
params={"velocity_range": {"x": (-0.5, 0.5), "y": (-0.5, 0.5)}},
|
|
),
|
|
}
|
|
|
|
rewards = {
|
|
"track_linear_velocity": RewardTermCfg(
|
|
func=mdp.track_linear_velocity,
|
|
weight=2.0,
|
|
params={"command_name": "twist", "std": math.sqrt(0.25)},
|
|
),
|
|
"dof_pos_limits": RewardTermCfg(func=mdp.joint_pos_limits, weight=-1.0),
|
|
"action_rate_l2": RewardTermCfg(func=mdp.action_rate_l2, weight=-0.1),
|
|
}
|
|
|
|
terminations = {
|
|
"time_out": TerminationTermCfg(func=mdp.time_out, time_out=True),
|
|
"fell_over": TerminationTermCfg(
|
|
func=mdp.bad_orientation,
|
|
params={"limit_angle": math.radians(70.0)},
|
|
),
|
|
}
|
|
|
|
return ManagerBasedRlEnvCfg(
|
|
decimation=4,
|
|
episode_length_s=20.0,
|
|
sim=SimulationCfg(
|
|
nconmax=35,
|
|
njmax=1500,
|
|
mujoco=MujocoCfg(timestep=0.005, iterations=10, ls_iterations=20),
|
|
),
|
|
scene=SceneCfg(
|
|
terrain=TerrainEntityCfg(
|
|
terrain_type="generator",
|
|
terrain_generator=replace(ROUGH_TERRAINS_CFG),
|
|
max_init_terrain_level=5,
|
|
),
|
|
num_envs=1,
|
|
),
|
|
observations=observations,
|
|
actions=actions,
|
|
commands=commands,
|
|
events=events,
|
|
rewards=rewards,
|
|
terminations=terminations,
|
|
)
|
|
|
|
Robot-specific configs call this factory and patch fields using
|
|
``dataclasses.replace`` or direct assignment. Common per-robot overrides
|
|
include ``scene`` (to add the robot entity and sensors), joint name patterns
|
|
inside ``SceneEntityCfg``, action ``scale``, and body names for reward terms.
|
|
|
|
.. note::
|
|
|
|
Isaac Lab uses deeply nested ``__post_init__`` overrides for configuration
|
|
inheritance. mjlab avoids that pattern: each ``ManagerBasedRlEnvCfg`` is a
|
|
flat, inspectable dataclass. A misspelled field name raises a ``TypeError``
|
|
at construction rather than silently creating a new attribute. See
|
|
:ref:`migration_isaac_lab` for a full comparison.
|
|
|
|
|
|
Where to go next
|
|
----------------
|
|
|
|
The remaining pages in the Manager Layer section cover each manager in
|
|
detail:
|
|
|
|
- :ref:`observations`: observation groups, the processing pipeline
|
|
(clip, scale, noise, delay, history), and built-in observation functions.
|
|
- :ref:`actions`: action types and how the action manager routes policy
|
|
output to actuators.
|
|
- :ref:`rewards`: reward terms and scaling by dt.
|
|
- :ref:`terminations`: episode end conditions and the truncation/failure
|
|
distinction.
|
|
- :ref:`commands`: command generators and goal-conditioned task setup.
|
|
- :ref:`events`: the event manager lifecycle (startup, reset, interval).
|
|
- :ref:`domain_randomization`: the full ``dr`` module for domain
|
|
randomization.
|
|
- :ref:`curriculum`: difficulty progression based on policy performance.
|
|
- :ref:`metrics`: custom per-step metrics logged as episode averages.
|