"""Watch a training run and play checkpoints as they are saved.""" import argparse import json import subprocess import time from pathlib import Path from playall import checkpoint_sort_key, play_checkpoint def has_video(checkpoint: Path, video_folder: Path) -> bool: return (video_folder / f"{checkpoint.stem}.mp4").is_file() or any( video_folder.glob(f"{checkpoint.stem}-*.mp4") ) def get_task_name(log_dir: Path, parser: argparse.ArgumentParser) -> str: run_file = log_dir / "run.json" try: with run_file.open(encoding="utf-8") as file: run_metadata = json.load(file) except FileNotFoundError: parser.error(f"Training metadata file not found: {run_file}") except json.JSONDecodeError as error: parser.error(f"Invalid JSON in {run_file}: {error}") except OSError as error: parser.error(f"Could not read {run_file}: {error}") if not isinstance(run_metadata, dict): parser.error(f"Expected a JSON object in {run_file}") task_name = run_metadata.get("task") if not isinstance(task_name, str) or not task_name.strip(): parser.error(f"Expected a non-empty 'task' string in {run_file}") return task_name.strip() def watch_run(log_dir: Path, task_name: str, interval: float, gpu: str) -> None: video_folder = log_dir / "videos" / "play" observed: dict[Path, tuple[int, int, int]] = {} print( f"Watching {log_dir} for checkpoints (task: {task_name}, " f"GPU: {gpu}, poll interval: {interval:g}s). Press Ctrl+C to stop.", flush=True, ) while True: checkpoints = sorted(log_dir.glob("*.pt"), key=checkpoint_sort_key) current_paths = set(checkpoints) for checkpoint in checkpoints: try: stat = checkpoint.stat() except FileNotFoundError: continue signature = (stat.st_size, stat.st_mtime_ns) previous = observed.get(checkpoint) stable_scans = previous[2] + 1 if previous and previous[:2] == signature else 1 observed[checkpoint] = (signature[0], signature[1], stable_scans) if stable_scans < 2 or stat.st_size == 0 or has_video(checkpoint, video_folder): continue try: play_checkpoint(checkpoint, video_folder, task_name, gpu) except subprocess.CalledProcessError as error: print( f"Playback failed for {checkpoint.name} with exit code {error.returncode}; " "will retry during the next scan.", flush=True, ) observed = {path: state for path, state in observed.items() if path in current_paths} time.sleep(interval) def main() -> None: parser = argparse.ArgumentParser( description="Continuously watch a training run and play new .pt checkpoints." ) parser.add_argument( "log_dir", nargs="?", help="Training run directory containing run.json and checkpoint .pt files.", ) parser.add_argument( "--interval", type=float, default=10.0, help="Seconds between directory scans (default: 10).", ) parser.add_argument( "--gpu", default="0", help="GPU index or CUDA_VISIBLE_DEVICES list to use (default: 0).", ) args = parser.parse_args() if args.interval <= 0: parser.error("--interval must be greater than zero") log_dir_input = args.log_dir or input("Training log directory: ").strip() log_dir = Path(log_dir_input).expanduser().resolve() if not log_dir.is_dir(): parser.error(f"Training log directory does not exist: {log_dir}") task_name = get_task_name(log_dir, parser) try: watch_run(log_dir, task_name, args.interval, args.gpu) except KeyboardInterrupt: print("\nAutoplay stopped.", flush=True) if __name__ == "__main__": main()