From a9a1199c220daa7a56b7284d9ad4efac837bc6ff Mon Sep 17 00:00:00 2001 From: zhenai Date: Sun, 4 Oct 2026 13:53:51 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9C=89=E4=B8=80=E4=B8=AA=E8=83=BD=E7=94=A8?= =?UTF-8?q?=E7=9A=84model=E7=94=A8=E6=9D=A5=E5=81=9As2r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tasks/manager_based/twofootbot/Lv0.py | 30 ++++++-- .../tasks/manager_based/twofootbot/Lv1.py | 74 ++++++++++--------- .../tasks/manager_based/twofootbot/Lv2.py | 2 +- .../tasks/manager_based/twofootbot/Lv3.py | 62 ++++++++++++++++ .../manager_based/twofootbot/__init__.py | 20 ++++- .../twofootbot/agents/rsl_rl_ppo_cfg.py | 4 +- .../manager_based/twofootbot/mdp/__init__.pyi | 2 + .../twofootbot/mdp/terminations.py | 56 ++++++++++++++ 8 files changed, 202 insertions(+), 48 deletions(-) create mode 100644 source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv3.py create mode 100644 source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/terminations.py diff --git a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv0.py b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv0.py index fa27f88..2058d54 100644 --- a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv0.py +++ b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv0.py @@ -245,6 +245,7 @@ class ObservationsCfg: """Observations for policy group.""" # observation terms (order preserved) + # 创建观测项，用于定义环境中的各种观测信息 # 每个观测项都包含一个函数和一个可选的噪声配置 # 基线线性速度观测项，添加均匀噪声，噪声范围在-0.1到0.1之间 @@ -281,7 +282,7 @@ class EventCfg: """Configuration for events.""" # reset -# 定义一个基础重置事件项，用于重置机器人的初始状态 +#定义一个基础重置事件项，用于重置机器人的初始状态 reset_base = EventTerm( # 指定重置函数为velocity_mdp模块中的reset_root_state_uniform函数 func=velocity_mdp.reset_root_state_uniform, @@ -308,10 +309,16 @@ class RewardsCfg: """Reward terms for the MDP.""" # (1) Constant running reward + # 定义跟踪角速度z方向的指数奖励项配置 + track_ang_vel_z_exp = RewTerm( + func=mdp.track_ang_vel_z_exp, # 奖励函数，用于跟踪角速度z方向的指数 + weight=2.0, # 奖励项权重，控制该奖励项的重要性 + params={"command_name": "base_velocity", "std": 0.25}, # 命令名称和标准差参数 + ) # 轨迹线性速度奖励项 track_lin_vel = RewTerm( # 定义一个奖励项，用于跟踪线性速度 func=velocity_mdp.track_lin_vel_xy_exp, # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法 - weight=5.0, + weight=5, params={"command_name": "base_velocity", "std": math.sqrt(0.25)}, ) # 创建奖励项，用于控制机器人的各种行为 @@ -325,19 +332,26 @@ class RewardsCfg: @configclass class TerminationsCfg: - """Termination terms for the MDP.""" - # (1) Time out + # 设置超时终止条件，当仿真运行时间超过设定值时终止 time_out = DoneTerm(func=mdp.time_out, time_out=True) - # (2) The robot has tipped too far from upright. + low_speed = DoneTerm( + func=mdp.root_low_speed_for_duration, + params={ + "asset_cfg": SceneEntityCfg("robot"), + "threshold": 0.1, + "duration": 1.0, + }, + ) + # 设置姿态终止条件，当机器人姿态超过限制角度时终止 fallen_orientation = DoneTerm( func=mdp.bad_orientation, - params={"limit_angle": 0.7}, + params={"limit_angle": 0.7}, # 限制角度设为0.7弧度 ) - # (3) The base has collapsed onto the ground. + # 设置高度终止条件，当机器人高度低于最小值时终止 fallen_height = DoneTerm( func=mdp.root_height_below_minimum, - params={"minimum_height": 0.10}, + params={"minimum_height": 0.10}, # 最小高度设为0.1米 ) diff --git a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv1.py b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv1.py index 6815ec7..d083bde 100644 --- a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv1.py +++ b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv1.py @@ -39,20 +39,26 @@ class Lv1RewardsCfg(RewardsCfg): # 定义跟踪角速度z方向的指数奖励项配置 # track_ang_vel_z_exp = RewardTermCfg( # func=mdp.track_ang_vel_z_exp, # 奖励函数，用于跟踪角速度z方向的指数 - # weight=0.5, # 奖励项权重，控制该奖励项的重要性 + # weight=2.5, # 奖励项权重，控制该奖励项的重要性 # params={"command_name": "base_velocity", "std": 0.25}, # 命令名称和标准差参数 # ) + # # 轨迹线性速度奖励项 + # track_lin_vel = RewTerm( # 定义一个奖励项，用于跟踪线性速度 + # func=mdp.track_lin_vel_xy_exp, # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法 + # weight=7.5, + # params={"command_name": "base_velocity", "std": math.sqrt(0.25)}, + # ) #奖励抬脚 - feet_air_time = RewTerm( - func=mdp.feet_air_time_positive_biped, # 调用的函数，用于计算双足机器人脚部空中时间 - weight=0.5, # 该奖励项的权重，表示在总奖励中的重要性 - params={ # 函数所需的参数配置 - "command_name": "base_velocity", # 命令名称，可能与基础速度控制相关 - "sensor_cfg": SceneEntityCfg("contact_forces", body_names=["lf3_link", "rf3_link"]), # 传感器配置，指定监测接触力的脚部刚体 - "threshold": 0.4, # 阈值，用于判断脚部是否接触地面 - }, - ) + # feet_air_time = RewTerm( + # func=mdp.feet_air_time_positive_biped, # 调用的函数，用于计算双足机器人脚部空中时间 + # weight=0.5, # 该奖励项的权重，表示在总奖励中的重要性 + # params={ # 函数所需的参数配置 + # "command_name": "base_velocity", # 命令名称，可能与基础速度控制相关 + # "sensor_cfg": SceneEntityCfg("contact_forces", body_names=["lf3_link", "rf3_link"]), # 传感器配置，指定监测接触力的脚部刚体 + # "threshold": 0.4, # 阈值，用于判断脚部是否接触地面 + # }, + # ) #打滑扣分 # feet_slide = RewTerm( # # 指定使用的函数为mdp.feet_slide @@ -68,36 +74,36 @@ class Lv1RewardsCfg(RewardsCfg): # }, # ) - # 定义奖励项配置：动作速率的L2范数惩罚 - action_rate_l2 = RewardTermCfg(func=mdp.action_rate_l2, weight=-5e-3) - # 定义奖励项配置：关节力矩最小化 - joint_torque_minimization = RewardTermCfg(func=mdp.joint_torques_l2, weight=-1e-4) - # 定义奖励项配置：关节速度最小化 - joint_velocity_minimization = RewardTermCfg(func=mdp.joint_vel_l2, weight=-1e-2) + # # 定义奖励项配置：动作速率的L2范数惩罚 + # action_rate_l2 = RewardTermCfg(func=mdp.action_rate_l2, weight=-5e-3) + # # 定义奖励项配置：关节力矩最小化 + # joint_torque_minimization = RewardTermCfg(func=mdp.joint_torques_l2, weight=-1e-4) + # # 定义奖励项配置：关节速度最小化 + # joint_velocity_minimization = RewardTermCfg(func=mdp.joint_vel_l2, weight=-1e-2) @configclass class Lv1CommandsCfg(CommandsCfg): - base_velocity = mdp.UniformVelocityCommandCfg( - # 资产名称设置为"robot" - asset_name="robot", - # 重采样时间范围设置为(10.0, 10.0) - resampling_time_range=(4.0, 6.0), - # 相对站立环境参数设置为0.02 - rel_standing_envs=0.02, - # 相对朝向环境参数设置为1.0 - rel_heading_envs=1.0, - # 启用朝向命令 - heading_command=False, - # 设置朝向控制刚度为0.5 - #heading_control_stiffness=0.5, - debug_vis=True, - ranges=mdp.UniformVelocityCommandCfg.Ranges( - lin_vel_x=(0.2, 0.5), lin_vel_y=(-0.0, 0.0), ang_vel_z=(-0.0, 0.0), heading=(-math.pi, math.pi) - ), - ) + base_velocity = mdp.UniformVelocityCommandCfg( + asset_name="robot", + resampling_time_range=(3.0, 6.0), + # 相对航向环境参数，用于控制或影响相对航向相关的环境配置 + # 该参数的默认值为0.0，表示在不设置特殊航向环境时的基准值 + rel_standing_envs=0.0, + # 相对航向环境参数，默认值为0.0 + rel_heading_envs=0.0, + # 航向命令标志，默认为False + heading_command=False, + debug_vis=True, + ranges=mdp.UniformVelocityCommandCfg.Ranges( + lin_vel_x=(0.2, 0.2), + lin_vel_y=(0.0, 0.0), + ang_vel_z=(-0.1, 0.1), + heading=(-math.pi, math.pi), + ), + ) @configclass diff --git a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv2.py b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv2.py index 671caca..02a1d19 100644 --- a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv2.py +++ b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv2.py @@ -79,7 +79,7 @@ class Lv2EventCfg(EventCfg): ) # interval - # 创建一个名为push_robot的EventTerm对象，用于执行机器人推动操作 + #创建一个名为push_robot的EventTerm对象，用于执行机器人推动操作 push_robot = EventTerm( func=mdp.push_by_setting_velocity, # 指定执行函数为mdp中的push_by_setting_velocity mode="interval", # 设置执行模式为间隔模式 diff --git a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv3.py b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv3.py new file mode 100644 index 0000000..3111105 --- /dev/null +++ b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv3.py @@ -0,0 +1,62 @@ +from isaaclab.managers import SceneEntityCfg +from isaaclab.utils.configclass import configclass + + +from TwoFootBot.tasks.manager_based.twofootbot.Lv2 import Lv2Cfg +from isaaclab.managers import ObservationGroupCfg as ObsGroup +from isaaclab.managers import ObservationTermCfg as ObsTerm +from isaaclab.utils.configclass import configclass +from isaaclab.utils.noise import UniformNoiseCfg + +import isaaclab_tasks.manager_based.locomotion.velocity.mdp as velocity_mdp + + +@configclass +class ObservationsCfg: + """Observation specifications for the MDP.""" + + @configclass + class PolicyCfg(ObsGroup): + """Observations for policy group.""" + + # observation terms (order preserved) +# 创建观测项，用于定义环境中的各种观测信息 +# 每个观测项都包含一个函数和一个可选的噪声配置 +# 基线线性速度观测项，添加均匀噪声，噪声范围在-0.1到0.1之间 + '''base_lin_vel = ObsTerm( + func=imu_lin_vel, + params={"asset_cfg": SceneEntityCfg("robot"), "offset_b": IMU_OFFSET_B}, + noise=UniformNoiseCfg(n_min=-0.1, n_max=0.1), + )''' +# 基线角速度观测项，添加均匀噪声，噪声范围在-0.2到0.2之间 + base_ang_vel = ObsTerm(func=velocity_mdp.base_ang_vel, noise=UniformNoiseCfg(n_min=-0.2, n_max=0.2)) +# 投影重力观测项，不添加噪声 + projected_gravity = ObsTerm(func=velocity_mdp.projected_gravity) +# 速度命令观测项，使用特定的命令名称"base_velocity" + velocity_commands = ObsTerm( + func=velocity_mdp.generated_commands, params={"command_name": "base_velocity"} + ) +# 相对关节位置观测项，使用场景中的机器人实体 + joint_pos_rel = ObsTerm(func=velocity_mdp.joint_pos_rel, params={"asset_cfg": SceneEntityCfg("robot")}) +# 相对关节速度观测项，使用场景中的机器人实体 + joint_vel_rel = ObsTerm(func=velocity_mdp.joint_vel_rel, params={"asset_cfg": SceneEntityCfg("robot")}) +# 上一个动作观测项 + actions = ObsTerm(func=velocity_mdp.last_action) + + def __post_init__(self) -> None: + self.enable_corruption = False + self.concatenate_terms = True + + # observation groups + policy: PolicyCfg = PolicyCfg() + +@configclass +class Lv3Cfg(Lv2Cfg): + """Configuration for the first level of the TwoFootBot task.""" + observations: ObservationsCfg = ObservationsCfg() + decimation = 4 + + + def __post_init__(self) -> None: + """Post initialization.""" + self.episode_length_s = 80.0 # time_out \ No newline at end of file diff --git a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/__init__.py b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/__init__.py index a611f63..e3958d4 100644 --- a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/__init__.py +++ b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/__init__.py @@ -51,8 +51,9 @@ gym.register( }, )''' - +#CUDA_VISIBLE_DEVICES=1 python ... 可把所有计算放在cuda1，直接是由--device cuda:1只会把torch放在1而sim在0 #$ python scripts/rsl_rl/train.py --task Template-Twofootbot-mlp-v0 +#robot 学会在理想情况下行走 v=0.2m/s gym.register( id="Template-Twofootbot-mlp-v0", entry_point="isaaclab.envs:ManagerBasedRLEnv", @@ -70,7 +71,7 @@ gym.register( # --resume \ # --load_run 2026-09-15_13-16-41-V0 \ # --checkpoint model_4999.pt --device cuda:0 - +#robot学会在变速情况下行走 v=0.2m/s--0.5m/s gym.register( id="Template-Twofootbot-mlp-v1", entry_point="isaaclab.envs:ManagerBasedRLEnv", @@ -82,7 +83,7 @@ gym.register( ) # Lv2Cfg - +#加入随机的重量，之心，外力干扰。 gym.register( id="Template-Twofootbot-mlp-v2", entry_point="isaaclab.envs:ManagerBasedRLEnv", @@ -91,4 +92,17 @@ gym.register( "env_cfg_entry_point": f"{__name__}.Lv2:Lv2Cfg", "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:PPORunnerCfg", }, +) + +# Lv3Cfg +# 尝试去除base_lin_vel这个在机器人上面不太好拿到的信息 + +gym.register( + id="Template-Twofootbot-mlp-v3", + entry_point="isaaclab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": f"{__name__}.Lv3:Lv3Cfg", + "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:PPORunnerCfg", + }, ) \ No newline at end of file diff --git a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/agents/rsl_rl_ppo_cfg.py b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/agents/rsl_rl_ppo_cfg.py index 81ade99..3369992 100644 --- a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/agents/rsl_rl_ppo_cfg.py +++ b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/agents/rsl_rl_ppo_cfg.py @@ -41,7 +41,7 @@ class PPORunnerCfg(RslRlOnPolicyRunnerCfg): distribution_cfg=RslRlMLPModelCfg.GaussianDistributionCfg(init_std=1.0), ) critic = RslRlMLPModelCfg( - hidden_dims=[1024, 512, 512, 256, 256, 128], #[1024, 512, 512, 256, 256, 128], + hidden_dims=[1024, 512, 512, 256, 256, 128], #[1024, 512, 512, 256, 256, 128], activation="relu", obs_normalization=True, ) @@ -53,7 +53,7 @@ class PPORunnerCfg(RslRlOnPolicyRunnerCfg): entropy_coef=0.005, #熵系数，探索的奖励 num_learning_epochs=5, #学习的轮数 num_mini_batches=4, #小批量的数量 - learning_rate=5.0e-4, #学习率 + learning_rate=1e-3, #学习率 schedule="adaptive", #学习率调度器类型 gamma=0.99, #折扣因子 lam=0.95, #GAE的lambda参数 diff --git a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/__init__.pyi b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/__init__.pyi index 8807c4a..898a0d3 100644 --- a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/__init__.pyi +++ b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/__init__.pyi @@ -5,9 +5,11 @@ __all__ = [ "joint_pos_target_l2", + "root_low_speed_for_duration", ] # Forward stable MDP terms lazily, then override with environment-specific terms below. from isaaclab.envs.mdp import * # noqa: F401, F403 from .rewards import joint_pos_target_l2 +from .terminations import root_low_speed_for_duration diff --git a/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/terminations.py b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/terminations.py new file mode 100644 index 0000000..4fef08e --- /dev/null +++ b/source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/terminations.py @@ -0,0 +1,56 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from isaaclab.managers import SceneEntityCfg + +if TYPE_CHECKING: + from isaaclab.envs import ManagerBasedRLEnv + + +def root_low_speed_for_duration( + env: ManagerBasedRLEnv, + threshold: float, + duration: float, + asset_cfg: SceneEntityCfg, +) -> torch.Tensor: + """Terminate environments whose robot stays below a planar speed threshold for a duration.""" + if threshold < 0.0: + raise ValueError(f"Speed threshold must be non-negative, got {threshold}.") + if duration <= 0.0: + raise ValueError(f"Duration must be positive, got {duration}.") + + asset = env.scene[asset_cfg.name] + speed = torch.linalg.vector_norm(asset.data.root_lin_vel_w[:, :2], dim=1) + episode_length = env.episode_length_buf + + elapsed = getattr(env, "_root_low_speed_elapsed", None) + last_episode_length = getattr(env, "_root_low_speed_last_episode_length", None) + if elapsed is None: + elapsed = torch.zeros_like(speed) + last_episode_length = episode_length.clone() + setattr(env, "_root_low_speed_elapsed", elapsed) + elif not isinstance(last_episode_length, torch.Tensor): + raise RuntimeError("Low-speed termination episode state is missing or invalid.") + + if elapsed.shape != speed.shape or elapsed.device != speed.device: + raise RuntimeError("Low-speed termination state does not match the environment.") + if last_episode_length.shape != episode_length.shape or last_episode_length.device != episode_length.device: + raise RuntimeError("Low-speed termination episode state does not match the environment.") + + elapsed[episode_length <= last_episode_length] = 0.0 + below_threshold = speed < threshold + elapsed[below_threshold] += env.step_dt + elapsed[~below_threshold] = 0.0 + + terminated = elapsed > duration + elapsed[terminated] = 0.0 + setattr(env, "_root_low_speed_last_episode_length", episode_length.clone()) + return terminated