117 lines
3.9 KiB
Python
117 lines
3.9 KiB
Python
"""Watch a training run and play checkpoints as they are saved."""
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from playall import checkpoint_sort_key, play_checkpoint
|
|
|
|
|
|
def has_video(checkpoint: Path, video_folder: Path) -> bool:
|
|
return (video_folder / f"{checkpoint.stem}.mp4").is_file() or any(
|
|
video_folder.glob(f"{checkpoint.stem}-*.mp4")
|
|
)
|
|
|
|
|
|
def get_task_name(log_dir: Path, parser: argparse.ArgumentParser) -> str:
|
|
run_file = log_dir / "run.json"
|
|
try:
|
|
with run_file.open(encoding="utf-8") as file:
|
|
run_metadata = json.load(file)
|
|
except FileNotFoundError:
|
|
parser.error(f"Training metadata file not found: {run_file}")
|
|
except json.JSONDecodeError as error:
|
|
parser.error(f"Invalid JSON in {run_file}: {error}")
|
|
except OSError as error:
|
|
parser.error(f"Could not read {run_file}: {error}")
|
|
|
|
if not isinstance(run_metadata, dict):
|
|
parser.error(f"Expected a JSON object in {run_file}")
|
|
task_name = run_metadata.get("task")
|
|
if not isinstance(task_name, str) or not task_name.strip():
|
|
parser.error(f"Expected a non-empty 'task' string in {run_file}")
|
|
return task_name.strip()
|
|
|
|
|
|
def watch_run(log_dir: Path, task_name: str, interval: float, gpu: str) -> None:
|
|
video_folder = log_dir / "videos" / "play"
|
|
observed: dict[Path, tuple[int, int, int]] = {}
|
|
print(
|
|
f"Watching {log_dir} for checkpoints (task: {task_name}, "
|
|
f"GPU: {gpu}, poll interval: {interval:g}s). Press Ctrl+C to stop.",
|
|
flush=True,
|
|
)
|
|
|
|
while True:
|
|
checkpoints = sorted(log_dir.glob("*.pt"), key=checkpoint_sort_key)
|
|
current_paths = set(checkpoints)
|
|
|
|
for checkpoint in checkpoints:
|
|
try:
|
|
stat = checkpoint.stat()
|
|
except FileNotFoundError:
|
|
continue
|
|
|
|
signature = (stat.st_size, stat.st_mtime_ns)
|
|
previous = observed.get(checkpoint)
|
|
stable_scans = previous[2] + 1 if previous and previous[:2] == signature else 1
|
|
observed[checkpoint] = (signature[0], signature[1], stable_scans)
|
|
|
|
if stable_scans < 2 or stat.st_size == 0 or has_video(checkpoint, video_folder):
|
|
continue
|
|
|
|
try:
|
|
play_checkpoint(checkpoint, video_folder, task_name, gpu)
|
|
except subprocess.CalledProcessError as error:
|
|
print(
|
|
f"Playback failed for {checkpoint.name} with exit code {error.returncode}; "
|
|
"will retry during the next scan.",
|
|
flush=True,
|
|
)
|
|
|
|
observed = {path: state for path, state in observed.items() if path in current_paths}
|
|
time.sleep(interval)
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(
|
|
description="Continuously watch a training run and play new .pt checkpoints."
|
|
)
|
|
parser.add_argument(
|
|
"log_dir",
|
|
nargs="?",
|
|
help="Training run directory containing run.json and checkpoint .pt files.",
|
|
)
|
|
parser.add_argument(
|
|
"--interval",
|
|
type=float,
|
|
default=10.0,
|
|
help="Seconds between directory scans (default: 10).",
|
|
)
|
|
parser.add_argument(
|
|
"--gpu",
|
|
default="0",
|
|
help="GPU index or CUDA_VISIBLE_DEVICES list to use (default: 0).",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
if args.interval <= 0:
|
|
parser.error("--interval must be greater than zero")
|
|
|
|
log_dir_input = args.log_dir or input("Training log directory: ").strip()
|
|
log_dir = Path(log_dir_input).expanduser().resolve()
|
|
if not log_dir.is_dir():
|
|
parser.error(f"Training log directory does not exist: {log_dir}")
|
|
|
|
task_name = get_task_name(log_dir, parser)
|
|
try:
|
|
watch_run(log_dir, task_name, args.interval, args.gpu)
|
|
except KeyboardInterrupt:
|
|
print("\nAutoplay stopped.", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|