有一个能用的model用来做s2r

source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv0.py

@@ -245,6 +245,7 @@ class ObservationsCfg:
         """Observations for policy group."""
 
         # observation terms (order preserved)
+       
 # 创建观测项，用于定义环境中的各种观测信息
 # 每个观测项都包含一个函数和一个可选的噪声配置
 # 基线线性速度观测项，添加均匀噪声，噪声范围在-0.1到0.1之间
@@ -281,7 +282,7 @@ class EventCfg:
     """Configuration for events."""
 
     # reset
-# 定义一个基础重置事件项，用于重置机器人的初始状态
+#定义一个基础重置事件项，用于重置机器人的初始状态
     reset_base = EventTerm(
     # 指定重置函数为velocity_mdp模块中的reset_root_state_uniform函数
         func=velocity_mdp.reset_root_state_uniform,
@@ -308,10 +309,16 @@ class RewardsCfg:
     """Reward terms for the MDP."""
 
     # (1) Constant running reward
+    # 定义跟踪角速度z方向的指数奖励项配置
+    track_ang_vel_z_exp = RewTerm(
+        func=mdp.track_ang_vel_z_exp,  # 奖励函数，用于跟踪角速度z方向的指数
+        weight=2.0,  # 奖励项权重，控制该奖励项的重要性
+        params={"command_name": "base_velocity", "std": 0.25},  # 命令名称和标准差参数
+    )
     # 轨迹线性速度奖励项
     track_lin_vel = RewTerm(  # 定义一个奖励项，用于跟踪线性速度
         func=velocity_mdp.track_lin_vel_xy_exp,  # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法
-        weight=5.0,
+        weight=5,
         params={"command_name": "base_velocity", "std": math.sqrt(0.25)},
     )
     # 创建奖励项，用于控制机器人的各种行为
@@ -325,19 +332,26 @@ class RewardsCfg:
 
 @configclass
 class TerminationsCfg:
-    """Termination terms for the MDP."""
 
-    # (1) Time out
+    # 设置超时终止条件，当仿真运行时间超过设定值时终止
     time_out = DoneTerm(func=mdp.time_out, time_out=True)
-    # (2) The robot has tipped too far from upright.
+    low_speed = DoneTerm(
+        func=mdp.root_low_speed_for_duration,
+        params={
+            "asset_cfg": SceneEntityCfg("robot"),
+            "threshold": 0.1,
+            "duration": 1.0,
+        },
+    )
+    # 设置姿态终止条件，当机器人姿态超过限制角度时终止
     fallen_orientation = DoneTerm(
         func=mdp.bad_orientation,
-        params={"limit_angle": 0.7},
+        params={"limit_angle": 0.7},  # 限制角度设为0.7弧度
     )
-    # (3) The base has collapsed onto the ground.
+    # 设置高度终止条件，当机器人高度低于最小值时终止
     fallen_height = DoneTerm(
         func=mdp.root_height_below_minimum,
-        params={"minimum_height": 0.10},
+        params={"minimum_height": 0.10},  # 最小高度设为0.1米
     )
 
 

source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv1.py

@@ -39,20 +39,26 @@ class Lv1RewardsCfg(RewardsCfg):
 # 定义跟踪角速度z方向的指数奖励项配置
     # track_ang_vel_z_exp = RewardTermCfg(
     #     func=mdp.track_ang_vel_z_exp,  # 奖励函数，用于跟踪角速度z方向的指数
-    #     weight=0.5,  # 奖励项权重，控制该奖励项的重要性
+    #     weight=2.5,  # 奖励项权重，控制该奖励项的重要性
     #     params={"command_name": "base_velocity", "std": 0.25},  # 命令名称和标准差参数
     # )
+    # # 轨迹线性速度奖励项
+    # track_lin_vel = RewTerm(  # 定义一个奖励项，用于跟踪线性速度
+    #     func=mdp.track_lin_vel_xy_exp,  # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法
+    #     weight=7.5,
+    #     params={"command_name": "base_velocity", "std": math.sqrt(0.25)},
+    # )
 
     #奖励抬脚
-    feet_air_time = RewTerm(
+    # feet_air_time = RewTerm(
-            func=mdp.feet_air_time_positive_biped,  # 调用的函数，用于计算双足机器人脚部空中时间
+    #         func=mdp.feet_air_time_positive_biped,  # 调用的函数，用于计算双足机器人脚部空中时间
-            weight=0.5,  # 该奖励项的权重，表示在总奖励中的重要性
+    #         weight=0.5,  # 该奖励项的权重，表示在总奖励中的重要性
-            params={  # 函数所需的参数配置
+    #         params={  # 函数所需的参数配置
-                "command_name": "base_velocity",  # 命令名称，可能与基础速度控制相关
+    #             "command_name": "base_velocity",  # 命令名称，可能与基础速度控制相关
-                "sensor_cfg": SceneEntityCfg("contact_forces", body_names=["lf3_link", "rf3_link"]),  # 传感器配置，指定监测接触力的脚部刚体
+    #             "sensor_cfg": SceneEntityCfg("contact_forces", body_names=["lf3_link", "rf3_link"]),  # 传感器配置，指定监测接触力的脚部刚体
-                "threshold": 0.4,  # 阈值，用于判断脚部是否接触地面
+    #             "threshold": 0.4,  # 阈值，用于判断脚部是否接触地面
-            },
+    #         },
-        )
+    #     )
     #打滑扣分
     # feet_slide = RewTerm(
     #         # 指定使用的函数为mdp.feet_slide
@@ -68,36 +74,36 @@ class Lv1RewardsCfg(RewardsCfg):
     #         },
     #     )
 
-    # 定义奖励项配置：动作速率的L2范数惩罚
+    # # 定义奖励项配置：动作速率的L2范数惩罚
-    action_rate_l2 = RewardTermCfg(func=mdp.action_rate_l2, weight=-5e-3)
+    # action_rate_l2 = RewardTermCfg(func=mdp.action_rate_l2, weight=-5e-3)
-    # 定义奖励项配置：关节力矩最小化
+    # # 定义奖励项配置：关节力矩最小化
-    joint_torque_minimization = RewardTermCfg(func=mdp.joint_torques_l2, weight=-1e-4)
+    # joint_torque_minimization = RewardTermCfg(func=mdp.joint_torques_l2, weight=-1e-4)
-    # 定义奖励项配置：关节速度最小化
+    # # 定义奖励项配置：关节速度最小化
-    joint_velocity_minimization = RewardTermCfg(func=mdp.joint_vel_l2, weight=-1e-2)
+    # joint_velocity_minimization = RewardTermCfg(func=mdp.joint_vel_l2, weight=-1e-2)
 
 
 
 
 @configclass
 class Lv1CommandsCfg(CommandsCfg):
-    base_velocity = mdp.UniformVelocityCommandCfg(
+      base_velocity = mdp.UniformVelocityCommandCfg(
-        # 资产名称设置为"robot"
+            asset_name="robot",
-        asset_name="robot",
+            resampling_time_range=(3.0, 6.0),
-        # 重采样时间范围设置为(10.0, 10.0)
+            # 相对航向环境参数，用于控制或影响相对航向相关的环境配置
-        resampling_time_range=(4.0, 6.0),
+            # 该参数的默认值为0.0，表示在不设置特殊航向环境时的基准值
-        # 相对站立环境参数设置为0.02
+            rel_standing_envs=0.0,
-        rel_standing_envs=0.02,
+            # 相对航向环境参数，默认值为0.0
-        # 相对朝向环境参数设置为1.0
+            rel_heading_envs=0.0,
-        rel_heading_envs=1.0,
+            # 航向命令标志，默认为False
-        # 启用朝向命令
+            heading_command=False,
-        heading_command=False,
+            debug_vis=True,
-        # 设置朝向控制刚度为0.5
+            ranges=mdp.UniformVelocityCommandCfg.Ranges(
-        #heading_control_stiffness=0.5,
+                lin_vel_x=(0.2, 0.2),
-        debug_vis=True,
+                lin_vel_y=(0.0, 0.0),
-        ranges=mdp.UniformVelocityCommandCfg.Ranges(
+                ang_vel_z=(-0.1, 0.1),
-            lin_vel_x=(0.2, 0.5), lin_vel_y=(-0.0, 0.0), ang_vel_z=(-0.0, 0.0), heading=(-math.pi, math.pi)
+                heading=(-math.pi, math.pi),
-        ),
+            ),
-    )
+        )
     
 
 @configclass

source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv2.py

@@ -79,7 +79,7 @@ class Lv2EventCfg(EventCfg):
         )
     
         # interval
-    # 创建一个名为push_robot的EventTerm对象，用于执行机器人推动操作
+    #创建一个名为push_robot的EventTerm对象，用于执行机器人推动操作
         push_robot = EventTerm(
             func=mdp.push_by_setting_velocity,  # 指定执行函数为mdp中的push_by_setting_velocity
             mode="interval",  # 设置执行模式为间隔模式

source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/Lv3.py

@@ -0,0 +1,62 @@
+from isaaclab.managers import SceneEntityCfg
+from isaaclab.utils.configclass import configclass
+
+
+from TwoFootBot.tasks.manager_based.twofootbot.Lv2 import Lv2Cfg
+from isaaclab.managers import ObservationGroupCfg as ObsGroup
+from isaaclab.managers import ObservationTermCfg as ObsTerm
+from isaaclab.utils.configclass import configclass
+from isaaclab.utils.noise import UniformNoiseCfg
+
+import isaaclab_tasks.manager_based.locomotion.velocity.mdp as velocity_mdp
+
+
+@configclass
+class ObservationsCfg:
+    """Observation specifications for the MDP."""
+
+    @configclass
+    class PolicyCfg(ObsGroup):
+        """Observations for policy group."""
+
+        # observation terms (order preserved)
+# 创建观测项，用于定义环境中的各种观测信息
+# 每个观测项都包含一个函数和一个可选的噪声配置
+# 基线线性速度观测项，添加均匀噪声，噪声范围在-0.1到0.1之间
+        '''base_lin_vel = ObsTerm(
+            func=imu_lin_vel,
+            params={"asset_cfg": SceneEntityCfg("robot"), "offset_b": IMU_OFFSET_B},
+            noise=UniformNoiseCfg(n_min=-0.1, n_max=0.1),
+        )'''
+# 基线角速度观测项，添加均匀噪声，噪声范围在-0.2到0.2之间
+        base_ang_vel = ObsTerm(func=velocity_mdp.base_ang_vel, noise=UniformNoiseCfg(n_min=-0.2, n_max=0.2))
+# 投影重力观测项，不添加噪声
+        projected_gravity = ObsTerm(func=velocity_mdp.projected_gravity)
+# 速度命令观测项，使用特定的命令名称"base_velocity"
+        velocity_commands = ObsTerm(
+            func=velocity_mdp.generated_commands, params={"command_name": "base_velocity"}
+        )
+# 相对关节位置观测项，使用场景中的机器人实体
+        joint_pos_rel = ObsTerm(func=velocity_mdp.joint_pos_rel, params={"asset_cfg": SceneEntityCfg("robot")})
+# 相对关节速度观测项，使用场景中的机器人实体
+        joint_vel_rel = ObsTerm(func=velocity_mdp.joint_vel_rel, params={"asset_cfg": SceneEntityCfg("robot")})
+# 上一个动作观测项
+        actions = ObsTerm(func=velocity_mdp.last_action)
+
+        def __post_init__(self) -> None:
+            self.enable_corruption = False
+            self.concatenate_terms = True
+
+    # observation groups
+    policy: PolicyCfg = PolicyCfg()
+
+@configclass
+class Lv3Cfg(Lv2Cfg):
+    """Configuration for the first level of the TwoFootBot task."""
+    observations: ObservationsCfg = ObservationsCfg()
+    decimation = 4
+    
+
+    def __post_init__(self) -> None:
+        """Post initialization."""
+        self.episode_length_s = 80.0  # time_out

source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/__init__.py

@@ -51,8 +51,9 @@ gym.register(
     },
 )'''
 
-
+#CUDA_VISIBLE_DEVICES=1 python ...   可把所有计算放在cuda1，直接是由--device cuda:1只会把torch放在1而sim在0
 #$ python scripts/rsl_rl/train.py --task Template-Twofootbot-mlp-v0
+#robot 学会在理想情况下行走 v=0.2m/s
 gym.register(
     id="Template-Twofootbot-mlp-v0",
     entry_point="isaaclab.envs:ManagerBasedRLEnv",
@@ -70,7 +71,7 @@ gym.register(
 #     --resume \
 #     --load_run 2026-09-15_13-16-41-V0 \
 #     --checkpoint model_4999.pt --device cuda:0             
-
+#robot学会在变速情况下行走 v=0.2m/s--0.5m/s
 gym.register(
     id="Template-Twofootbot-mlp-v1",
     entry_point="isaaclab.envs:ManagerBasedRLEnv",
@@ -82,7 +83,7 @@ gym.register(
 )
 
 # Lv2Cfg
-
+#加入随机的重量，之心，外力干扰。
 gym.register(
     id="Template-Twofootbot-mlp-v2",
     entry_point="isaaclab.envs:ManagerBasedRLEnv",
@@ -91,4 +92,17 @@ gym.register(
         "env_cfg_entry_point": f"{__name__}.Lv2:Lv2Cfg",
         "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:PPORunnerCfg",
     },
+)
+
+# Lv3Cfg
+# 尝试去除base_lin_vel这个在机器人上面不太好拿到的信息
+
+gym.register(
+    id="Template-Twofootbot-mlp-v3",
+    entry_point="isaaclab.envs:ManagerBasedRLEnv",
+    disable_env_checker=True,
+    kwargs={
+        "env_cfg_entry_point": f"{__name__}.Lv3:Lv3Cfg",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:PPORunnerCfg",
+    },
 )

source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/agents/rsl_rl_ppo_cfg.py

@@ -41,7 +41,7 @@ class PPORunnerCfg(RslRlOnPolicyRunnerCfg):
         distribution_cfg=RslRlMLPModelCfg.GaussianDistributionCfg(init_std=1.0),
     )
     critic = RslRlMLPModelCfg(
-        hidden_dims=[1024, 512, 512, 256, 256, 128],                    #[1024, 512, 512, 256, 256, 128],
+        hidden_dims=[1024, 512, 512, 256, 256, 128],                      #[1024, 512, 512, 256, 256, 128],
         activation="relu",
         obs_normalization=True,
     )
@@ -53,7 +53,7 @@ class PPORunnerCfg(RslRlOnPolicyRunnerCfg):
         entropy_coef=0.005, #熵系数，探索的奖励
         num_learning_epochs=5, #学习的轮数
         num_mini_batches=4, #小批量的数量
-        learning_rate=5.0e-4, #学习率
+        learning_rate=1e-3, #学习率
         schedule="adaptive", #学习率调度器类型
         gamma=0.99, #折扣因子
         lam=0.95, #GAE的lambda参数

source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/__init__.pyi

@@ -5,9 +5,11 @@
 
 __all__ = [
     "joint_pos_target_l2",
+    "root_low_speed_for_duration",
 ]
 
 # Forward stable MDP terms lazily, then override with environment-specific terms below.
 from isaaclab.envs.mdp import *  # noqa: F401, F403
 
 from .rewards import joint_pos_target_l2
+from .terminations import root_low_speed_for_duration

source/TwoFootBot/TwoFootBot/tasks/manager_based/twofootbot/mdp/terminations.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import torch
+
+from isaaclab.managers import SceneEntityCfg
+
+if TYPE_CHECKING:
+    from isaaclab.envs import ManagerBasedRLEnv
+
+
+def root_low_speed_for_duration(
+    env: ManagerBasedRLEnv,
+    threshold: float,
+    duration: float,
+    asset_cfg: SceneEntityCfg,
+) -> torch.Tensor:
+    """Terminate environments whose robot stays below a planar speed threshold for a duration."""
+    if threshold < 0.0:
+        raise ValueError(f"Speed threshold must be non-negative, got {threshold}.")
+    if duration <= 0.0:
+        raise ValueError(f"Duration must be positive, got {duration}.")
+
+    asset = env.scene[asset_cfg.name]
+    speed = torch.linalg.vector_norm(asset.data.root_lin_vel_w[:, :2], dim=1)
+    episode_length = env.episode_length_buf
+
+    elapsed = getattr(env, "_root_low_speed_elapsed", None)
+    last_episode_length = getattr(env, "_root_low_speed_last_episode_length", None)
+    if elapsed is None:
+        elapsed = torch.zeros_like(speed)
+        last_episode_length = episode_length.clone()
+        setattr(env, "_root_low_speed_elapsed", elapsed)
+    elif not isinstance(last_episode_length, torch.Tensor):
+        raise RuntimeError("Low-speed termination episode state is missing or invalid.")
+
+    if elapsed.shape != speed.shape or elapsed.device != speed.device:
+        raise RuntimeError("Low-speed termination state does not match the environment.")
+    if last_episode_length.shape != episode_length.shape or last_episode_length.device != episode_length.device:
+        raise RuntimeError("Low-speed termination episode state does not match the environment.")
+
+    elapsed[episode_length <= last_episode_length] = 0.0
+    below_threshold = speed < threshold
+    elapsed[below_threshold] += env.step_dt
+    elapsed[~below_threshold] = 0.0
+
+    terminated = elapsed > duration
+    elapsed[terminated] = 0.0
+    setattr(env, "_root_low_speed_last_episode_length", episode_length.clone())
+    return terminated