有一个能用的model用来做s2r
This commit is contained in:
parent
f2194a2a2b
commit
a9a1199c22
|
|
@ -245,6 +245,7 @@ class ObservationsCfg:
|
|||
"""Observations for policy group."""
|
||||
|
||||
# observation terms (order preserved)
|
||||
|
||||
# 创建观测项，用于定义环境中的各种观测信息
|
||||
# 每个观测项都包含一个函数和一个可选的噪声配置
|
||||
# 基线线性速度观测项，添加均匀噪声，噪声范围在-0.1到0.1之间
|
||||
|
|
@ -281,7 +282,7 @@ class EventCfg:
|
|||
"""Configuration for events."""
|
||||
|
||||
# reset
|
||||
# 定义一个基础重置事件项，用于重置机器人的初始状态
|
||||
#定义一个基础重置事件项，用于重置机器人的初始状态
|
||||
reset_base = EventTerm(
|
||||
# 指定重置函数为velocity_mdp模块中的reset_root_state_uniform函数
|
||||
func=velocity_mdp.reset_root_state_uniform,
|
||||
|
|
@ -308,10 +309,16 @@ class RewardsCfg:
|
|||
"""Reward terms for the MDP."""
|
||||
|
||||
# (1) Constant running reward
|
||||
# 定义跟踪角速度z方向的指数奖励项配置
|
||||
track_ang_vel_z_exp = RewTerm(
|
||||
func=mdp.track_ang_vel_z_exp, # 奖励函数，用于跟踪角速度z方向的指数
|
||||
weight=2.0, # 奖励项权重，控制该奖励项的重要性
|
||||
params={"command_name": "base_velocity", "std": 0.25}, # 命令名称和标准差参数
|
||||
)
|
||||
# 轨迹线性速度奖励项
|
||||
track_lin_vel = RewTerm( # 定义一个奖励项，用于跟踪线性速度
|
||||
func=velocity_mdp.track_lin_vel_xy_exp, # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法
|
||||
weight=5.0,
|
||||
weight=5,
|
||||
params={"command_name": "base_velocity", "std": math.sqrt(0.25)},
|
||||
)
|
||||
# 创建奖励项，用于控制机器人的各种行为
|
||||
|
|
@ -325,19 +332,26 @@ class RewardsCfg:
|
|||
|
||||
@configclass
|
||||
class TerminationsCfg:
|
||||
"""Termination terms for the MDP."""
|
||||
|
||||
# (1) Time out
|
||||
# 设置超时终止条件，当仿真运行时间超过设定值时终止
|
||||
time_out = DoneTerm(func=mdp.time_out, time_out=True)
|
||||
# (2) The robot has tipped too far from upright.
|
||||
low_speed = DoneTerm(
|
||||
func=mdp.root_low_speed_for_duration,
|
||||
params={
|
||||
"asset_cfg": SceneEntityCfg("robot"),
|
||||
"threshold": 0.1,
|
||||
"duration": 1.0,
|
||||
},
|
||||
)
|
||||
# 设置姿态终止条件，当机器人姿态超过限制角度时终止
|
||||
fallen_orientation = DoneTerm(
|
||||
func=mdp.bad_orientation,
|
||||
params={"limit_angle": 0.7},
|
||||
params={"limit_angle": 0.7}, # 限制角度设为0.7弧度
|
||||
)
|
||||
# (3) The base has collapsed onto the ground.
|
||||
# 设置高度终止条件，当机器人高度低于最小值时终止
|
||||
fallen_height = DoneTerm(
|
||||
func=mdp.root_height_below_minimum,
|
||||
params={"minimum_height": 0.10},
|
||||
params={"minimum_height": 0.10}, # 最小高度设为0.1米
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -39,20 +39,26 @@ class Lv1RewardsCfg(RewardsCfg):
|
|||
# 定义跟踪角速度z方向的指数奖励项配置
|
||||
# track_ang_vel_z_exp = RewardTermCfg(
|
||||
# func=mdp.track_ang_vel_z_exp, # 奖励函数，用于跟踪角速度z方向的指数
|
||||
# weight=0.5, # 奖励项权重，控制该奖励项的重要性
|
||||
# weight=2.5, # 奖励项权重，控制该奖励项的重要性
|
||||
# params={"command_name": "base_velocity", "std": 0.25}, # 命令名称和标准差参数
|
||||
# )
|
||||
# # 轨迹线性速度奖励项
|
||||
# track_lin_vel = RewTerm( # 定义一个奖励项，用于跟踪线性速度
|
||||
# func=mdp.track_lin_vel_xy_exp, # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法
|
||||
# weight=7.5,
|
||||
# params={"command_name": "base_velocity", "std": math.sqrt(0.25)},
|
||||
# )
|
||||
|
||||
#奖励抬脚
|
||||
feet_air_time = RewTerm(
|
||||
func=mdp.feet_air_time_positive_biped, # 调用的函数，用于计算双足机器人脚部空中时间
|
||||
weight=0.5, # 该奖励项的权重，表示在总奖励中的重要性
|
||||
params={ # 函数所需的参数配置
|
||||
"command_name": "base_velocity", # 命令名称，可能与基础速度控制相关
|
||||
"sensor_cfg": SceneEntityCfg("contact_forces", body_names=["lf3_link", "rf3_link"]), # 传感器配置，指定监测接触力的脚部刚体
|
||||
"threshold": 0.4, # 阈值，用于判断脚部是否接触地面
|
||||
},
|
||||
)
|
||||
# feet_air_time = RewTerm(
|
||||
# func=mdp.feet_air_time_positive_biped, # 调用的函数，用于计算双足机器人脚部空中时间
|
||||
# weight=0.5, # 该奖励项的权重，表示在总奖励中的重要性
|
||||
# params={ # 函数所需的参数配置
|
||||
# "command_name": "base_velocity", # 命令名称，可能与基础速度控制相关
|
||||
# "sensor_cfg": SceneEntityCfg("contact_forces", body_names=["lf3_link", "rf3_link"]), # 传感器配置，指定监测接触力的脚部刚体
|
||||
# "threshold": 0.4, # 阈值，用于判断脚部是否接触地面
|
||||
# },
|
||||
# )
|
||||
#打滑扣分
|
||||
# feet_slide = RewTerm(
|
||||
# # 指定使用的函数为mdp.feet_slide
|
||||
|
|
@ -68,12 +74,12 @@ class Lv1RewardsCfg(RewardsCfg):
|
|||
# },
|
||||
# )
|
||||
|
||||
# 定义奖励项配置：动作速率的L2范数惩罚
|
||||
action_rate_l2 = RewardTermCfg(func=mdp.action_rate_l2, weight=-5e-3)
|
||||
# 定义奖励项配置：关节力矩最小化
|
||||
joint_torque_minimization = RewardTermCfg(func=mdp.joint_torques_l2, weight=-1e-4)
|
||||
# 定义奖励项配置：关节速度最小化
|
||||
joint_velocity_minimization = RewardTermCfg(func=mdp.joint_vel_l2, weight=-1e-2)
|
||||
# # 定义奖励项配置：动作速率的L2范数惩罚
|
||||
# action_rate_l2 = RewardTermCfg(func=mdp.action_rate_l2, weight=-5e-3)
|
||||
# # 定义奖励项配置：关节力矩最小化
|
||||
# joint_torque_minimization = RewardTermCfg(func=mdp.joint_torques_l2, weight=-1e-4)
|
||||
# # 定义奖励项配置：关节速度最小化
|
||||
# joint_velocity_minimization = RewardTermCfg(func=mdp.joint_vel_l2, weight=-1e-2)
|
||||
|
||||
|
||||
|
||||
|
|
@ -81,21 +87,21 @@ class Lv1RewardsCfg(RewardsCfg):
|
|||
@configclass
|
||||
class Lv1CommandsCfg(CommandsCfg):
|
||||
base_velocity = mdp.UniformVelocityCommandCfg(
|
||||
# 资产名称设置为"robot"
|
||||
asset_name="robot",
|
||||
# 重采样时间范围设置为(10.0, 10.0)
|
||||
resampling_time_range=(4.0, 6.0),
|
||||
# 相对站立环境参数设置为0.02
|
||||
rel_standing_envs=0.02,
|
||||
# 相对朝向环境参数设置为1.0
|
||||
rel_heading_envs=1.0,
|
||||
# 启用朝向命令
|
||||
resampling_time_range=(3.0, 6.0),
|
||||
# 相对航向环境参数，用于控制或影响相对航向相关的环境配置
|
||||
# 该参数的默认值为0.0，表示在不设置特殊航向环境时的基准值
|
||||
rel_standing_envs=0.0,
|
||||
# 相对航向环境参数，默认值为0.0
|
||||
rel_heading_envs=0.0,
|
||||
# 航向命令标志，默认为False
|
||||
heading_command=False,
|
||||
# 设置朝向控制刚度为0.5
|
||||
#heading_control_stiffness=0.5,
|
||||
debug_vis=True,
|
||||
ranges=mdp.UniformVelocityCommandCfg.Ranges(
|
||||
lin_vel_x=(0.2, 0.5), lin_vel_y=(-0.0, 0.0), ang_vel_z=(-0.0, 0.0), heading=(-math.pi, math.pi)
|
||||
lin_vel_x=(0.2, 0.2),
|
||||
lin_vel_y=(0.0, 0.0),
|
||||
ang_vel_z=(-0.1, 0.1),
|
||||
heading=(-math.pi, math.pi),
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ class Lv2EventCfg(EventCfg):
|
|||
)
|
||||
|
||||
# interval
|
||||
# 创建一个名为push_robot的EventTerm对象，用于执行机器人推动操作
|
||||
#创建一个名为push_robot的EventTerm对象，用于执行机器人推动操作
|
||||
push_robot = EventTerm(
|
||||
func=mdp.push_by_setting_velocity, # 指定执行函数为mdp中的push_by_setting_velocity
|
||||
mode="interval", # 设置执行模式为间隔模式
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
from isaaclab.managers import SceneEntityCfg
|
||||
from isaaclab.utils.configclass import configclass
|
||||
|
||||
|
||||
from TwoFootBot.tasks.manager_based.twofootbot.Lv2 import Lv2Cfg
|
||||
from isaaclab.managers import ObservationGroupCfg as ObsGroup
|
||||
from isaaclab.managers import ObservationTermCfg as ObsTerm
|
||||
from isaaclab.utils.configclass import configclass
|
||||
from isaaclab.utils.noise import UniformNoiseCfg
|
||||
|
||||
import isaaclab_tasks.manager_based.locomotion.velocity.mdp as velocity_mdp
|
||||
|
||||
|
||||
@configclass
|
||||
class ObservationsCfg:
|
||||
"""Observation specifications for the MDP."""
|
||||
|
||||
@configclass
|
||||
class PolicyCfg(ObsGroup):
|
||||
"""Observations for policy group."""
|
||||
|
||||
# observation terms (order preserved)
|
||||
# 创建观测项，用于定义环境中的各种观测信息
|
||||
# 每个观测项都包含一个函数和一个可选的噪声配置
|
||||
# 基线线性速度观测项，添加均匀噪声，噪声范围在-0.1到0.1之间
|
||||
'''base_lin_vel = ObsTerm(
|
||||
func=imu_lin_vel,
|
||||
params={"asset_cfg": SceneEntityCfg("robot"), "offset_b": IMU_OFFSET_B},
|
||||
noise=UniformNoiseCfg(n_min=-0.1, n_max=0.1),
|
||||
)'''
|
||||
# 基线角速度观测项，添加均匀噪声，噪声范围在-0.2到0.2之间
|
||||
base_ang_vel = ObsTerm(func=velocity_mdp.base_ang_vel, noise=UniformNoiseCfg(n_min=-0.2, n_max=0.2))
|
||||
# 投影重力观测项，不添加噪声
|
||||
projected_gravity = ObsTerm(func=velocity_mdp.projected_gravity)
|
||||
# 速度命令观测项，使用特定的命令名称"base_velocity"
|
||||
velocity_commands = ObsTerm(
|
||||
func=velocity_mdp.generated_commands, params={"command_name": "base_velocity"}
|
||||
)
|
||||
# 相对关节位置观测项，使用场景中的机器人实体
|
||||
joint_pos_rel = ObsTerm(func=velocity_mdp.joint_pos_rel, params={"asset_cfg": SceneEntityCfg("robot")})
|
||||
# 相对关节速度观测项，使用场景中的机器人实体
|
||||
joint_vel_rel = ObsTerm(func=velocity_mdp.joint_vel_rel, params={"asset_cfg": SceneEntityCfg("robot")})
|
||||
# 上一个动作观测项
|
||||
actions = ObsTerm(func=velocity_mdp.last_action)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.enable_corruption = False
|
||||
self.concatenate_terms = True
|
||||
|
||||
# observation groups
|
||||
policy: PolicyCfg = PolicyCfg()
|
||||
|
||||
@configclass
|
||||
class Lv3Cfg(Lv2Cfg):
|
||||
"""Configuration for the first level of the TwoFootBot task."""
|
||||
observations: ObservationsCfg = ObservationsCfg()
|
||||
decimation = 4
|
||||
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Post initialization."""
|
||||
self.episode_length_s = 80.0 # time_out
|
||||
|
|
@ -51,8 +51,9 @@ gym.register(
|
|||
},
|
||||
)'''
|
||||
|
||||
|
||||
#CUDA_VISIBLE_DEVICES=1 python ... 可把所有计算放在cuda1，直接是由--device cuda:1只会把torch放在1而sim在0
|
||||
#$ python scripts/rsl_rl/train.py --task Template-Twofootbot-mlp-v0
|
||||
#robot 学会在理想情况下行走 v=0.2m/s
|
||||
gym.register(
|
||||
id="Template-Twofootbot-mlp-v0",
|
||||
entry_point="isaaclab.envs:ManagerBasedRLEnv",
|
||||
|
|
@ -70,7 +71,7 @@ gym.register(
|
|||
# --resume \
|
||||
# --load_run 2026-09-15_13-16-41-V0 \
|
||||
# --checkpoint model_4999.pt --device cuda:0
|
||||
|
||||
#robot学会在变速情况下行走 v=0.2m/s--0.5m/s
|
||||
gym.register(
|
||||
id="Template-Twofootbot-mlp-v1",
|
||||
entry_point="isaaclab.envs:ManagerBasedRLEnv",
|
||||
|
|
@ -82,7 +83,7 @@ gym.register(
|
|||
)
|
||||
|
||||
# Lv2Cfg
|
||||
|
||||
#加入随机的重量，之心，外力干扰。
|
||||
gym.register(
|
||||
id="Template-Twofootbot-mlp-v2",
|
||||
entry_point="isaaclab.envs:ManagerBasedRLEnv",
|
||||
|
|
@ -92,3 +93,16 @@ gym.register(
|
|||
"rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:PPORunnerCfg",
|
||||
},
|
||||
)
|
||||
|
||||
# Lv3Cfg
|
||||
# 尝试去除base_lin_vel这个在机器人上面不太好拿到的信息
|
||||
|
||||
gym.register(
|
||||
id="Template-Twofootbot-mlp-v3",
|
||||
entry_point="isaaclab.envs:ManagerBasedRLEnv",
|
||||
disable_env_checker=True,
|
||||
kwargs={
|
||||
"env_cfg_entry_point": f"{__name__}.Lv3:Lv3Cfg",
|
||||
"rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:PPORunnerCfg",
|
||||
},
|
||||
)
|
||||
|
|
@ -53,7 +53,7 @@ class PPORunnerCfg(RslRlOnPolicyRunnerCfg):
|
|||
entropy_coef=0.005, #熵系数，探索的奖励
|
||||
num_learning_epochs=5, #学习的轮数
|
||||
num_mini_batches=4, #小批量的数量
|
||||
learning_rate=5.0e-4, #学习率
|
||||
learning_rate=1e-3, #学习率
|
||||
schedule="adaptive", #学习率调度器类型
|
||||
gamma=0.99, #折扣因子
|
||||
lam=0.95, #GAE的lambda参数
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@
|
|||
|
||||
__all__ = [
|
||||
"joint_pos_target_l2",
|
||||
"root_low_speed_for_duration",
|
||||
]
|
||||
|
||||
# Forward stable MDP terms lazily, then override with environment-specific terms below.
|
||||
from isaaclab.envs.mdp import * # noqa: F401, F403
|
||||
|
||||
from .rewards import joint_pos_target_l2
|
||||
from .terminations import root_low_speed_for_duration
|
||||
|
|
|
|||
|
|
@ -0,0 +1,56 @@
|
|||
# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
|
||||
# All rights reserved.
|
||||
#
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from isaaclab.managers import SceneEntityCfg
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from isaaclab.envs import ManagerBasedRLEnv
|
||||
|
||||
|
||||
def root_low_speed_for_duration(
|
||||
env: ManagerBasedRLEnv,
|
||||
threshold: float,
|
||||
duration: float,
|
||||
asset_cfg: SceneEntityCfg,
|
||||
) -> torch.Tensor:
|
||||
"""Terminate environments whose robot stays below a planar speed threshold for a duration."""
|
||||
if threshold < 0.0:
|
||||
raise ValueError(f"Speed threshold must be non-negative, got {threshold}.")
|
||||
if duration <= 0.0:
|
||||
raise ValueError(f"Duration must be positive, got {duration}.")
|
||||
|
||||
asset = env.scene[asset_cfg.name]
|
||||
speed = torch.linalg.vector_norm(asset.data.root_lin_vel_w[:, :2], dim=1)
|
||||
episode_length = env.episode_length_buf
|
||||
|
||||
elapsed = getattr(env, "_root_low_speed_elapsed", None)
|
||||
last_episode_length = getattr(env, "_root_low_speed_last_episode_length", None)
|
||||
if elapsed is None:
|
||||
elapsed = torch.zeros_like(speed)
|
||||
last_episode_length = episode_length.clone()
|
||||
setattr(env, "_root_low_speed_elapsed", elapsed)
|
||||
elif not isinstance(last_episode_length, torch.Tensor):
|
||||
raise RuntimeError("Low-speed termination episode state is missing or invalid.")
|
||||
|
||||
if elapsed.shape != speed.shape or elapsed.device != speed.device:
|
||||
raise RuntimeError("Low-speed termination state does not match the environment.")
|
||||
if last_episode_length.shape != episode_length.shape or last_episode_length.device != episode_length.device:
|
||||
raise RuntimeError("Low-speed termination episode state does not match the environment.")
|
||||
|
||||
elapsed[episode_length <= last_episode_length] = 0.0
|
||||
below_threshold = speed < threshold
|
||||
elapsed[below_threshold] += env.step_dt
|
||||
elapsed[~below_threshold] = 0.0
|
||||
|
||||
terminated = elapsed > duration
|
||||
elapsed[terminated] = 0.0
|
||||
setattr(env, "_root_low_speed_last_episode_length", episode_length.clone())
|
||||
return terminated
|
||||
Loading…
Reference in New Issue