2026-11-02 05:56:55 +00:00
|
|
|
"""Play every RSL-RL checkpoint in a training run directory."""
|
|
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
|
import json
|
|
|
|
|
import os
|
|
|
|
|
import re
|
|
|
|
|
import subprocess
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def checkpoint_sort_key(checkpoint: Path) -> tuple[int, int | str]:
|
|
|
|
|
match = re.fullmatch(r"model_(\d+)", checkpoint.stem)
|
|
|
|
|
if match:
|
|
|
|
|
return (0, int(match.group(1)))
|
|
|
|
|
return (1, checkpoint.name)
|
|
|
|
|
|
|
|
|
|
|
2026-11-09 08:34:58 +00:00
|
|
|
def play_checkpoint(checkpoint: Path, video_folder: Path, task_name: str, gpu: str = "0") -> None:
|
2026-11-02 05:56:55 +00:00
|
|
|
video_folder.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
expected_video = video_folder / f"{checkpoint.stem}.mp4"
|
|
|
|
|
if expected_video.exists():
|
|
|
|
|
if expected_video.is_file():
|
|
|
|
|
print(f"Skipping {checkpoint.name}; video already exists: {expected_video}")
|
|
|
|
|
return
|
|
|
|
|
raise FileExistsError(f"Expected video path exists but is not a file: {expected_video}")
|
|
|
|
|
|
|
|
|
|
existing_videos = {
|
|
|
|
|
path: (path.stat().st_mtime_ns, path.stat().st_size)
|
|
|
|
|
for path in video_folder.glob("*.mp4")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
command = [
|
|
|
|
|
"uv",
|
|
|
|
|
"run",
|
|
|
|
|
"--extra",
|
|
|
|
|
"video",
|
|
|
|
|
"isaaclab",
|
|
|
|
|
"play",
|
|
|
|
|
"--rl_library",
|
|
|
|
|
"rsl_rl",
|
|
|
|
|
"--task",
|
|
|
|
|
task_name,
|
|
|
|
|
"--num_envs",
|
|
|
|
|
"96",
|
|
|
|
|
"--checkpoint",
|
|
|
|
|
str(checkpoint),
|
|
|
|
|
"--video",
|
|
|
|
|
"--video_length",
|
2026-11-09 08:34:58 +00:00
|
|
|
"1000",
|
2026-11-02 05:56:55 +00:00
|
|
|
]
|
|
|
|
|
environment = os.environ.copy()
|
2026-11-09 08:34:58 +00:00
|
|
|
environment["CUDA_VISIBLE_DEVICES"] = gpu
|
2026-11-02 05:56:55 +00:00
|
|
|
|
|
|
|
|
print(f"\n=== Playing {checkpoint.name} to {task_name} ===", flush=True)
|
|
|
|
|
subprocess.run(command, cwd=PROJECT_ROOT, env=environment, check=True)
|
|
|
|
|
|
|
|
|
|
recorded_videos = sorted(
|
|
|
|
|
path
|
|
|
|
|
for path in video_folder.glob("*.mp4")
|
|
|
|
|
if path not in existing_videos
|
|
|
|
|
or (path.stat().st_mtime_ns, path.stat().st_size) != existing_videos[path]
|
|
|
|
|
)
|
|
|
|
|
if not recorded_videos:
|
|
|
|
|
raise RuntimeError(f"No video was recorded for {checkpoint}.")
|
|
|
|
|
|
|
|
|
|
for video in recorded_videos:
|
|
|
|
|
suffix = f"-{video.stem}" if len(recorded_videos) > 1 else ""
|
|
|
|
|
destination = video_folder / f"{checkpoint.stem}{suffix}.mp4"
|
|
|
|
|
if destination.exists():
|
|
|
|
|
raise FileExistsError(f"Refusing to overwrite existing video: {destination}")
|
|
|
|
|
video.rename(destination)
|
|
|
|
|
print(f"Saved video: {destination}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main() -> None:
|
|
|
|
|
parser = argparse.ArgumentParser(
|
|
|
|
|
description="Play all .pt checkpoints directly inside a training log directory."
|
|
|
|
|
)
|
|
|
|
|
parser.add_argument(
|
|
|
|
|
"log_dir",
|
|
|
|
|
nargs="?",
|
|
|
|
|
help="Training run directory containing checkpoint .pt files.",
|
|
|
|
|
)
|
2026-11-09 08:34:58 +00:00
|
|
|
parser.add_argument(
|
|
|
|
|
"--gpu",
|
|
|
|
|
default="0",
|
|
|
|
|
help="GPU index or CUDA_VISIBLE_DEVICES list to use (default: 0).",
|
|
|
|
|
)
|
2026-11-02 05:56:55 +00:00
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
|
|
|
|
log_dir_input = args.log_dir or input("Training log directory: ").strip()
|
|
|
|
|
log_dir = Path(log_dir_input).expanduser().resolve()
|
|
|
|
|
if not log_dir.is_dir():
|
|
|
|
|
parser.error(f"Training log directory does not exist: {log_dir}")
|
|
|
|
|
|
|
|
|
|
run_file = log_dir / "run.json"
|
|
|
|
|
try:
|
|
|
|
|
with run_file.open(encoding="utf-8") as file:
|
|
|
|
|
run_metadata = json.load(file)
|
|
|
|
|
except FileNotFoundError:
|
|
|
|
|
parser.error(f"Training metadata file not found: {run_file}")
|
|
|
|
|
except json.JSONDecodeError as error:
|
|
|
|
|
parser.error(f"Invalid JSON in {run_file}: {error}")
|
|
|
|
|
except OSError as error:
|
|
|
|
|
parser.error(f"Could not read {run_file}: {error}")
|
|
|
|
|
|
|
|
|
|
if not isinstance(run_metadata, dict):
|
|
|
|
|
parser.error(f"Expected a JSON object in {run_file}")
|
|
|
|
|
task_name = run_metadata.get("task")
|
|
|
|
|
if not isinstance(task_name, str) or not task_name.strip():
|
|
|
|
|
parser.error(f"Expected a non-empty 'task' string in {run_file}")
|
|
|
|
|
task_name = task_name.strip()
|
|
|
|
|
|
|
|
|
|
checkpoints = sorted(log_dir.glob("*.pt"), key=checkpoint_sort_key)
|
|
|
|
|
if not checkpoints:
|
|
|
|
|
parser.error(f"No .pt checkpoint files found directly inside: {log_dir}")
|
|
|
|
|
|
|
|
|
|
video_folder = log_dir / "videos" / "play"
|
|
|
|
|
pending_checkpoints = [
|
|
|
|
|
checkpoint
|
|
|
|
|
for checkpoint in checkpoints
|
|
|
|
|
if not (video_folder / f"{checkpoint.stem}.mp4").is_file()
|
|
|
|
|
]
|
|
|
|
|
skipped_count = len(checkpoints) - len(pending_checkpoints)
|
|
|
|
|
print(
|
|
|
|
|
f"Found {len(checkpoints)} checkpoint(s) for task {task_name} in {log_dir}; "
|
|
|
|
|
f"{skipped_count} already have videos, {len(pending_checkpoints)} to play."
|
|
|
|
|
)
|
|
|
|
|
for checkpoint in pending_checkpoints:
|
2026-11-09 08:34:58 +00:00
|
|
|
play_checkpoint(checkpoint, video_folder, task_name, args.gpu)
|
2026-11-02 05:56:55 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
main()
|