have a good model

This commit is contained in:
zhenai 2026-11-09 16:34:58 +08:00
parent d6cc7bbb53
commit d962687b9d
26 changed files with 255 additions and 25 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

9
CanGoModel/run.json Normal file
View File

@ -0,0 +1,9 @@
{
"created_at": "2026-11-06T03:42:24.021826+00:00",
"library": "rsl_rl",
"metadata": {
"agent": "rsl_rl_cfg_entry_point"
},
"task": "Template-Twofootbot-mlp-v2",
"version": 1
}

View File

@ -72,6 +72,21 @@ video. Videos are saved in the run's `videos/play` directory using the checkpoin
so rerunning the script only plays checkpoints that do not yet have a video. The script reports an error if `run.json` so rerunning the script only plays checkpoints that do not yet have a video. The script reports an error if `run.json`
is missing, invalid, or does not contain a non-empty `task`. is missing, invalid, or does not contain a non-empty `task`.
To keep watching a run while training and automatically play newly saved checkpoints, use `autoplay.py`:
```bash
uv run python autoplay.py logs/rsl_rl/cartpole_direct/<RUN_DIRECTORY>
# Adjust the scan interval if needed; default is 10 seconds.
uv run python autoplay.py logs/rsl_rl/cartpole_direct/<RUN_DIRECTORY> --interval 5
# Select GPU 1 instead of the default GPU 0.
uv run python autoplay.py logs/rsl_rl/cartpole_direct/<RUN_DIRECTORY> --gpu 1
```
The watcher waits for a checkpoint file to remain unchanged across scans before playing it, skips checkpoints that
already have a video, and can be stopped with `Ctrl+C`. `--gpu` sets `CUDA_VISIBLE_DEVICES` for the playback process;
it accepts one GPU index (for example `1`) or a comma-separated list (for example `0,1`). The same option is available
in `playall.py`.
## Resume an existing run ## Resume an existing run
The original project logs are intentionally not copied. Resume directly from its checkpoint while running from this The original project logs are intentionally not copied. Resume directly from its checkpoint while running from this

116
autoplay.py Normal file
View File

@ -0,0 +1,116 @@
"""Watch a training run and play checkpoints as they are saved."""
import argparse
import json
import subprocess
import time
from pathlib import Path
from playall import checkpoint_sort_key, play_checkpoint
def has_video(checkpoint: Path, video_folder: Path) -> bool:
return (video_folder / f"{checkpoint.stem}.mp4").is_file() or any(
video_folder.glob(f"{checkpoint.stem}-*.mp4")
)
def get_task_name(log_dir: Path, parser: argparse.ArgumentParser) -> str:
run_file = log_dir / "run.json"
try:
with run_file.open(encoding="utf-8") as file:
run_metadata = json.load(file)
except FileNotFoundError:
parser.error(f"Training metadata file not found: {run_file}")
except json.JSONDecodeError as error:
parser.error(f"Invalid JSON in {run_file}: {error}")
except OSError as error:
parser.error(f"Could not read {run_file}: {error}")
if not isinstance(run_metadata, dict):
parser.error(f"Expected a JSON object in {run_file}")
task_name = run_metadata.get("task")
if not isinstance(task_name, str) or not task_name.strip():
parser.error(f"Expected a non-empty 'task' string in {run_file}")
return task_name.strip()
def watch_run(log_dir: Path, task_name: str, interval: float, gpu: str) -> None:
video_folder = log_dir / "videos" / "play"
observed: dict[Path, tuple[int, int, int]] = {}
print(
f"Watching {log_dir} for checkpoints (task: {task_name}, "
f"GPU: {gpu}, poll interval: {interval:g}s). Press Ctrl+C to stop.",
flush=True,
)
while True:
checkpoints = sorted(log_dir.glob("*.pt"), key=checkpoint_sort_key)
current_paths = set(checkpoints)
for checkpoint in checkpoints:
try:
stat = checkpoint.stat()
except FileNotFoundError:
continue
signature = (stat.st_size, stat.st_mtime_ns)
previous = observed.get(checkpoint)
stable_scans = previous[2] + 1 if previous and previous[:2] == signature else 1
observed[checkpoint] = (signature[0], signature[1], stable_scans)
if stable_scans < 2 or stat.st_size == 0 or has_video(checkpoint, video_folder):
continue
try:
play_checkpoint(checkpoint, video_folder, task_name, gpu)
except subprocess.CalledProcessError as error:
print(
f"Playback failed for {checkpoint.name} with exit code {error.returncode}; "
"will retry during the next scan.",
flush=True,
)
observed = {path: state for path, state in observed.items() if path in current_paths}
time.sleep(interval)
def main() -> None:
parser = argparse.ArgumentParser(
description="Continuously watch a training run and play new .pt checkpoints."
)
parser.add_argument(
"log_dir",
nargs="?",
help="Training run directory containing run.json and checkpoint .pt files.",
)
parser.add_argument(
"--interval",
type=float,
default=10.0,
help="Seconds between directory scans (default: 10).",
)
parser.add_argument(
"--gpu",
default="0",
help="GPU index or CUDA_VISIBLE_DEVICES list to use (default: 0).",
)
args = parser.parse_args()
if args.interval <= 0:
parser.error("--interval must be greater than zero")
log_dir_input = args.log_dir or input("Training log directory: ").strip()
log_dir = Path(log_dir_input).expanduser().resolve()
if not log_dir.is_dir():
parser.error(f"Training log directory does not exist: {log_dir}")
task_name = get_task_name(log_dir, parser)
try:
watch_run(log_dir, task_name, args.interval, args.gpu)
except KeyboardInterrupt:
print("\nAutoplay stopped.", flush=True)
if __name__ == "__main__":
main()

View File

@ -17,7 +17,7 @@ def checkpoint_sort_key(checkpoint: Path) -> tuple[int, int | str]:
return (1, checkpoint.name) return (1, checkpoint.name)
def play_checkpoint(checkpoint: Path, video_folder: Path, task_name: str) -> None: def play_checkpoint(checkpoint: Path, video_folder: Path, task_name: str, gpu: str = "0") -> None:
video_folder.mkdir(parents=True, exist_ok=True) video_folder.mkdir(parents=True, exist_ok=True)
expected_video = video_folder / f"{checkpoint.stem}.mp4" expected_video = video_folder / f"{checkpoint.stem}.mp4"
if expected_video.exists(): if expected_video.exists():
@ -48,10 +48,10 @@ def play_checkpoint(checkpoint: Path, video_folder: Path, task_name: str) -> Non
str(checkpoint), str(checkpoint),
"--video", "--video",
"--video_length", "--video_length",
"500", "1000",
] ]
environment = os.environ.copy() environment = os.environ.copy()
environment["CUDA_VISIBLE_DEVICES"] = "1" environment["CUDA_VISIBLE_DEVICES"] = gpu
print(f"\n=== Playing {checkpoint.name} to {task_name} ===", flush=True) print(f"\n=== Playing {checkpoint.name} to {task_name} ===", flush=True)
subprocess.run(command, cwd=PROJECT_ROOT, env=environment, check=True) subprocess.run(command, cwd=PROJECT_ROOT, env=environment, check=True)
@ -83,6 +83,11 @@ def main() -> None:
nargs="?", nargs="?",
help="Training run directory containing checkpoint .pt files.", help="Training run directory containing checkpoint .pt files.",
) )
parser.add_argument(
"--gpu",
default="0",
help="GPU index or CUDA_VISIBLE_DEVICES list to use (default: 0).",
)
args = parser.parse_args() args = parser.parse_args()
log_dir_input = args.log_dir or input("Training log directory: ").strip() log_dir_input = args.log_dir or input("Training log directory: ").strip()
@ -124,7 +129,7 @@ def main() -> None:
f"{skipped_count} already have videos, {len(pending_checkpoints)} to play." f"{skipped_count} already have videos, {len(pending_checkpoints)} to play."
) )
for checkpoint in pending_checkpoints: for checkpoint in pending_checkpoints:
play_checkpoint(checkpoint, video_folder, task_name) play_checkpoint(checkpoint, video_folder, task_name, args.gpu)
if __name__ == "__main__": if __name__ == "__main__":

View File

@ -193,7 +193,11 @@ class TwofootbotdemoSceneCfg(InteractiveSceneCfg):
# 机器人配置:使用URDF配置文件,并设置其在场景中的路径 # 机器人配置:使用URDF配置文件,并设置其在场景中的路径
robot: ArticulationCfg = TWOFOOTBOT_URDF_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") robot: ArticulationCfg = TWOFOOTBOT_URDF_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot")
contact_forces = None # 接触力设置为None,表示不计算接触力 contact_forces = ContactSensorCfg(
prim_path="{ENV_REGEX_NS}/Robot/.*",
history_length=3,
track_air_time=True,
)
# 穹顶光源配置:设置场景中的全局光照 # 穹顶光源配置:设置场景中的全局光照
dome_light = AssetBaseCfg( dome_light = AssetBaseCfg(
@ -317,19 +321,28 @@ class RewardsCfg:
track_ang_vel_z_exp = RewTerm( track_ang_vel_z_exp = RewTerm(
func=mdp.track_ang_vel_z_exp, # 奖励函数,用于跟踪角速度z方向的指数 func=mdp.track_ang_vel_z_exp, # 奖励函数,用于跟踪角速度z方向的指数
weight=2.0, # 奖励项权重,控制该奖励项的重要性 weight=2.0, # 奖励项权重,控制该奖励项的重要性
params={"command_name": "base_velocity", "std": 0.5}, # 命令名称和标准差参数 params={"command_name": "base_velocity", "std": 0.1}, # 命令名称和标准差参数
) )
# 轨迹线性速度奖励项 # 轨迹线性速度奖励项
track_lin_vel = RewTerm( # 定义一个奖励项,用于跟踪线性速度 track_lin_vel = RewTerm( # 定义一个奖励项,用于跟踪线性速度
func=velocity_mdp.track_lin_vel_xy_exp, # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法 func=velocity_mdp.track_lin_vel_xy_exp, # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法
weight=5.0, weight=5.0,
params={"command_name": "base_velocity", "std": 0.5}, params={"command_name": "base_velocity", "std": 0.1},
)
feet_gait = RewTerm(
func=velocity_mdp.feet_air_time_positive_biped,
weight=1,
params={
"command_name": "base_velocity",
"threshold": 0.2,
"sensor_cfg": SceneEntityCfg("contact_forces", body_names=["lf3_link", "rf3_link"]),
},
) )
# 创建奖励项,用于控制机器人的各种行为 # 创建奖励项,用于控制机器人的各种行为
upright = RewTerm(func=velocity_mdp.flat_orientation_l2, weight=-1e-4) # 控制机器人保持直立姿态,权重为-1.0 upright = RewTerm(func=velocity_mdp.flat_orientation_l2, weight=-0.25) # 控制机器人保持直立姿态,权重为-1.0
vertical_velocity = RewTerm(func=velocity_mdp.lin_vel_z_l2, weight=-0.1) # 控制机器人垂直方向速度,权重为-1.0 vertical_velocity = RewTerm(func=velocity_mdp.lin_vel_z_l2, weight=-0.1) # 控制机器人垂直方向速度,权重为-1.0
angular_velocity = RewTerm(func=velocity_mdp.ang_vel_xy_l2, weight=-0.05) # 控制机器人角速度,权重为-0.05 angular_velocity = RewTerm(func=velocity_mdp.ang_vel_xy_l2, weight=-0.1) # 控制机器人角速度,权重为-0.05
action_rate = RewTerm(func=velocity_mdp.action_rate_l2, weight=-0.001) # 控制机器人动作变化率,权重为-0.01 action_rate = RewTerm(func=velocity_mdp.action_rate_l2, weight=-0.01) # 控制机器人动作变化率,权重为-0.01
joint_acceleration = RewTerm(func=velocity_mdp.joint_acc_l2, weight=-2.5e-7) # 控制机器人关节加速度,权重为-2.5e-7 joint_acceleration = RewTerm(func=velocity_mdp.joint_acc_l2, weight=-2.5e-7) # 控制机器人关节加速度,权重为-2.5e-7
joint_limits = RewTerm(func=velocity_mdp.joint_pos_limits, weight=-1.0) # 确保机器人关节位置在限制范围内,权重为-1.0 joint_limits = RewTerm(func=velocity_mdp.joint_pos_limits, weight=-1.0) # 确保机器人关节位置在限制范围内,权重为-1.0

View File

@ -110,13 +110,13 @@ class RewardsCfg(RewardsCfg):
track_ang_vel_z_exp = RewTerm( track_ang_vel_z_exp = RewTerm(
func=mdp.track_ang_vel_z_exp, # 奖励函数,用于跟踪角速度z方向的指数 func=mdp.track_ang_vel_z_exp, # 奖励函数,用于跟踪角速度z方向的指数
weight=2.0, # 奖励项权重,控制该奖励项的重要性 weight=2.0, # 奖励项权重,控制该奖励项的重要性
params={"command_name": "base_velocity", "std": 0.5}, # 命令名称和标准差参数 params={"command_name": "base_velocity", "std": 0.1}, # 命令名称和标准差参数
) )
# 轨迹线性速度奖励项 # 轨迹线性速度奖励项
track_lin_vel = RewTerm( # 定义一个奖励项,用于跟踪线性速度 track_lin_vel = RewTerm( # 定义一个奖励项,用于跟踪线性速度
func=mdp.track_lin_vel_xy_exp, # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法 func=mdp.track_lin_vel_xy_exp, # 使用velocity_mdp模块中的track_lin_vel_xy_exp函数作为奖励计算方法
weight=4.0, weight=5.0,
params={"command_name": "base_velocity", "std": 0.5}, params={"command_name": "base_velocity", "std": 0.1},
) )
@configclass @configclass

View File

@ -0,0 +1,44 @@
from isaaclab.managers import SceneEntityCfg
from isaaclab.utils.configclass import configclass
from isaaclab.managers import EventTermCfg as EventTerm
from isaaclab_physx.physics import PhysxCfg
import isaaclab_tasks.core.velocity.mdp as mdp
from TwoFootBot_isaaclab.tasks.manager_based.twofootbot.Lv2 import Lv2Cfg, Lv2EventCfg
@configclass
class Lv3EventCfg(Lv2EventCfg):
# 推动机器人的事件配置
push_robot = EventTerm(
func=mdp.push_by_setting_velocity, # 设置速度来推动机器人的函数
mode="interval", # 事件模式为间隔触发
interval_range_s=(10.0, 10.0), # 触发间隔时间范围(秒)
params={"velocity_range": {"x": (-0.1, 0.1), "y": (-0.1, 0.1)}}, # 推动速度范围,x和y方向的速度范围
)
# 随机化PD增益的事件配置
randomize_pd_gains = EventTerm(
func=mdp.randomize_actuator_gains, # 随机化执行器增益的函数
mode="startup", # 事件模式为启动时触发
params={
"asset_cfg": SceneEntityCfg("robot", joint_names=".*"), # 场景实体配置,机器人所有关节
"stiffness_distribution_params": (0.7, 1.3), # 刚度分布参数
"damping_distribution_params": (0.7, 1.3), # 阻尼分布参数
"operation": "scale", # 操作类型为缩放
"distribution": "uniform", # 分布类型为均匀分布
},
)
@configclass
class Lv3Cfg(Lv2Cfg):
"""Configuration for the third level of the TwoFootBot task."""
# scene: Lv3SceneCfg = Lv3SceneCfg(num_envs=4096 * 5, env_spacing=0.5)
events: Lv3EventCfg = Lv3EventCfg()
decimation = 4
def __post_init__(self) -> None:
"""Post initialization."""
self.episode_length_s = 120.0 #time_out
self.viewer.eye = (3.0, -3.0, 0.1)
self.viewer.lookat = (0.0, 0.0, 0.0)
self.sim.dt = 1 / 200
self.sim.physics = PhysxCfg(gpu_max_rigid_patch_count=100 * 2**15)

View File

@ -92,6 +92,16 @@ gym.register(
}, },
) )
gym.register(
id="Template-Twofootbot-mlp-v3",
entry_point="isaaclab.envs:ManagerBasedRLEnv",
disable_env_checker=True,
kwargs={
"env_cfg_entry_point": f"{__name__}.Lv3:Lv3Cfg",
"rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:PPORunnerCfg",
},
)
# # Lv2Cfg # # Lv2Cfg
# #加入随机的重量,之心,外力干扰。 # #加入随机的重量,之心,外力干扰。
# gym.register( # gym.register(

View File

@ -19,11 +19,11 @@ from isaaclab_rl.rsl_rl import (
class PPORunnerCfg(RslRlOnPolicyRunnerCfg): class PPORunnerCfg(RslRlOnPolicyRunnerCfg):
num_steps_per_env = 24 num_steps_per_env = 24
max_iterations = 50000 max_iterations = 50000
save_interval = 1000 save_interval = 200
experiment_name = "cartpole_direct" experiment_name = "cartpole_direct"
#obs_groups = {"actor": ["policy", "camera"], "critic": ["policy"]} #obs_groups = {"actor": ["policy", "camera"], "critic": ["policy"]}
obs_groups = {"actor": ["policy"], "critic": ["policy"], "rnd_state": ["policy"]} #obs_groups = {"actor": ["policy"], "critic": ["policy"], "rnd_state": ["policy"]}
#obs_groups = {"actor": ["policy"], "critic": ["policy"]} obs_groups = {"actor": ["policy"], "critic": ["policy"]}
'''actor = RslRlCNNModelCfg( '''actor = RslRlCNNModelCfg(
hidden_dims=[512, 256, 256, 128], hidden_dims=[512, 256, 256, 128],
activation="relu", activation="relu",
@ -63,12 +63,12 @@ class PPORunnerCfg(RslRlOnPolicyRunnerCfg):
desired_kl=0.01, #期望的KL散度 desired_kl=0.01, #期望的KL散度
max_grad_norm=1.0, #梯度裁剪的最大范数 max_grad_norm=1.0, #梯度裁剪的最大范数
# 创建随机强化学习配置对象 # 创建随机强化学习配置对象
rnd_cfg=RslRlRndCfg( # rnd_cfg=RslRlRndCfg(
weight=0.01, # 设置权重参数为0.01 # weight=0.01, # 设置权重参数为0.01
reward_normalization=True, # 启用奖励归一化 # reward_normalization=True, # 启用奖励归一化
state_normalization=True, # 启用状态归一化 # state_normalization=True, # 启用状态归一化
learning_rate=1e-3, # 设置学习率为0.001 # learning_rate=1e-3, # 设置学习率为0.001
predictor_hidden_dims=[128, 64], # 设置预测器隐藏层维度为[128, 64] # predictor_hidden_dims=[128, 64], # 设置预测器隐藏层维度为[128, 64]
target_hidden_dims=[128, 64], # 设置目标网络隐藏层维度为[128, 64] # target_hidden_dims=[128, 64], # 设置目标网络隐藏层维度为[128, 64]
), # ),
) )

View File

@ -3,6 +3,7 @@
# #
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
#DE_BUD
from isaaclab.managers import RewardTermCfg as RewTerm from isaaclab.managers import RewardTermCfg as RewTerm
from isaaclab.managers import SceneEntityCfg from isaaclab.managers import SceneEntityCfg

View File

@ -2,6 +2,7 @@
# All rights reserved. # All rights reserved.
# #
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
#DE_BUD
from isaaclab.managers import RewardTermCfg as RewTerm from isaaclab.managers import RewardTermCfg as RewTerm

View File

@ -2,6 +2,8 @@
使用 heading_command=True,转向由 heading 目标平滑驱动。 使用 heading_command=True,转向由 heading 目标平滑驱动。
""" """
#DE_BUD
from __future__ import annotations from __future__ import annotations

View File

@ -2,6 +2,7 @@
# All rights reserved. # All rights reserved.
# #
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
#DE_BUD
import math import math

View File

@ -2,6 +2,7 @@
# All rights reserved. # All rights reserved.
# #
# SPDX-License-Identifier: BSD-3-Clause # SPDX-License-Identifier: BSD-3-Clause
#DE_BUD
import math import math
from dataclasses import MISSING from dataclasses import MISSING

View File

@ -5,6 +5,8 @@
"""Tests for the project's task registrations.""" """Tests for the project's task registrations."""
import importlib
import gymnasium as gym import gymnasium as gym
import TwoFootBot_isaaclab.tasks # noqa: F401 import TwoFootBot_isaaclab.tasks # noqa: F401
@ -23,7 +25,17 @@ def test_task_registrations():
"TwoFootBot_isaaclab.tasks.manager_based.twofootbot.Lv1:Lv1Cfg", "TwoFootBot_isaaclab.tasks.manager_based.twofootbot.Lv1:Lv1Cfg",
"TwoFootBot_isaaclab.tasks.manager_based.twofootbot.agents.rsl_rl_ppo_cfg:PPORunnerCfg", "TwoFootBot_isaaclab.tasks.manager_based.twofootbot.agents.rsl_rl_ppo_cfg:PPORunnerCfg",
), ),
(
"Template-Twofootbot-mlp-v2",
"TwoFootBot_isaaclab.tasks.manager_based.twofootbot.Lv2:Lv2Cfg",
"TwoFootBot_isaaclab.tasks.manager_based.twofootbot.agents.rsl_rl_ppo_cfg:PPORunnerCfg",
),
): ):
spec = gym.spec(task_id) spec = gym.spec(task_id)
assert spec.kwargs["env_cfg_entry_point"] == env_cfg assert spec.kwargs["env_cfg_entry_point"] == env_cfg
assert spec.kwargs["rsl_rl_cfg_entry_point"] == agent_cfg assert spec.kwargs["rsl_rl_cfg_entry_point"] == agent_cfg
lv2_module = importlib.import_module(
"TwoFootBot_isaaclab.tasks.manager_based.twofootbot.Lv2"
)
assert hasattr(lv2_module, "Lv2Cfg")