TwoFootBot-isaaclab/autoplay.py

117 lines
3.9 KiB
Python
Raw Permalink Normal View History

2026-11-09 08:34:58 +00:00
"""Watch a training run and play checkpoints as they are saved."""
import argparse
import json
import subprocess
import time
from pathlib import Path
from playall import checkpoint_sort_key, play_checkpoint
def has_video(checkpoint: Path, video_folder: Path) -> bool:
return (video_folder / f"{checkpoint.stem}.mp4").is_file() or any(
video_folder.glob(f"{checkpoint.stem}-*.mp4")
)
def get_task_name(log_dir: Path, parser: argparse.ArgumentParser) -> str:
run_file = log_dir / "run.json"
try:
with run_file.open(encoding="utf-8") as file:
run_metadata = json.load(file)
except FileNotFoundError:
parser.error(f"Training metadata file not found: {run_file}")
except json.JSONDecodeError as error:
parser.error(f"Invalid JSON in {run_file}: {error}")
except OSError as error:
parser.error(f"Could not read {run_file}: {error}")
if not isinstance(run_metadata, dict):
parser.error(f"Expected a JSON object in {run_file}")
task_name = run_metadata.get("task")
if not isinstance(task_name, str) or not task_name.strip():
parser.error(f"Expected a non-empty 'task' string in {run_file}")
return task_name.strip()
def watch_run(log_dir: Path, task_name: str, interval: float, gpu: str) -> None:
video_folder = log_dir / "videos" / "play"
observed: dict[Path, tuple[int, int, int]] = {}
print(
f"Watching {log_dir} for checkpoints (task: {task_name}, "
f"GPU: {gpu}, poll interval: {interval:g}s). Press Ctrl+C to stop.",
flush=True,
)
while True:
checkpoints = sorted(log_dir.glob("*.pt"), key=checkpoint_sort_key)
current_paths = set(checkpoints)
for checkpoint in checkpoints:
try:
stat = checkpoint.stat()
except FileNotFoundError:
continue
signature = (stat.st_size, stat.st_mtime_ns)
previous = observed.get(checkpoint)
stable_scans = previous[2] + 1 if previous and previous[:2] == signature else 1
observed[checkpoint] = (signature[0], signature[1], stable_scans)
if stable_scans < 2 or stat.st_size == 0 or has_video(checkpoint, video_folder):
continue
try:
play_checkpoint(checkpoint, video_folder, task_name, gpu)
except subprocess.CalledProcessError as error:
print(
f"Playback failed for {checkpoint.name} with exit code {error.returncode}; "
"will retry during the next scan.",
flush=True,
)
observed = {path: state for path, state in observed.items() if path in current_paths}
time.sleep(interval)
def main() -> None:
parser = argparse.ArgumentParser(
description="Continuously watch a training run and play new .pt checkpoints."
)
parser.add_argument(
"log_dir",
nargs="?",
help="Training run directory containing run.json and checkpoint .pt files.",
)
parser.add_argument(
"--interval",
type=float,
default=10.0,
help="Seconds between directory scans (default: 10).",
)
parser.add_argument(
"--gpu",
default="0",
help="GPU index or CUDA_VISIBLE_DEVICES list to use (default: 0).",
)
args = parser.parse_args()
if args.interval <= 0:
parser.error("--interval must be greater than zero")
log_dir_input = args.log_dir or input("Training log directory: ").strip()
log_dir = Path(log_dir_input).expanduser().resolve()
if not log_dir.is_dir():
parser.error(f"Training log directory does not exist: {log_dir}")
task_name = get_task_name(log_dir, parser)
try:
watch_run(log_dir, task_name, args.interval, args.gpu)
except KeyboardInterrupt:
print("\nAutoplay stopped.", flush=True)
if __name__ == "__main__":
main()