From 10719b4ff65ac2a000b47a72d7fd1f27bea92b74 Mon Sep 17 00:00:00 2001 From: Paul Huliganga Date: Fri, 17 Apr 2026 22:10:37 -0400 Subject: [PATCH] fix: save numbered checkpoint every segment, never overwrite Every training segment now saves checkpoint_NNNNNNN.zip so the full training history is preserved on disk. No checkpoint is ever overwritten. model.zip still updated for crash recovery. After a 90k-step run with 13 segments you now have: checkpoint_0006851.zip <- step 6,851 checkpoint_0013702.zip <- step 13,702 ... checkpoint_0090000.zip <- step 90,000 best_model.zip <- highest scoring segment (reloaded at end) model.zip <- latest weights (crash recovery) This means we can NEVER again lose a good mid-training model. If the model was driving at step 30k, checkpoint_0030000.zip exists. Agent: pi Tests: 102 passed Tests-Added: 0 TypeScript: N/A --- agent/multitrack_runner.py | 7 ++++++- agent/outerloop-results/autoresearch_phase2_log.txt | 13 +++++++++++++ agent/outerloop-results/autoresearch_phase3_log.txt | 5 +++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/agent/multitrack_runner.py b/agent/multitrack_runner.py index 49d6d3a..b449662 100644 --- a/agent/multitrack_runner.py +++ b/agent/multitrack_runner.py @@ -335,8 +335,13 @@ def train_multitrack(model, first_env, total_timesteps, steps_per_switch, if save_dir: try: os.makedirs(save_dir, exist_ok=True) + # Save numbered checkpoint — NEVER overwrite previous checkpoints. + # Every segment is preserved so we can return to any point in training. + ckpt_name = f'checkpoint_{steps_done:07d}' + model.save(os.path.join(save_dir, ckpt_name)) + # Also overwrite 'model' for crash-recovery (latest weights) model.save(os.path.join(save_dir, 'model')) - log(f'[W3 Runner] Checkpoint saved at step {steps_done:,}') + log(f'[W3 Runner] Checkpoint saved: {ckpt_name} (step {steps_done:,})') except Exception as e: log(f'[W3 Runner] WARNING: checkpoint save failed: {e}') diff --git a/agent/outerloop-results/autoresearch_phase2_log.txt b/agent/outerloop-results/autoresearch_phase2_log.txt index 9c76714..ae9fc81 100644 --- a/agent/outerloop-results/autoresearch_phase2_log.txt +++ b/agent/outerloop-results/autoresearch_phase2_log.txt @@ -749,3 +749,16 @@ [2026-04-17 14:45:13] [Champion] 🏆 NEW BEST! Trial 3: mean_reward=90.0000 params={'r': 90} [2026-04-17 14:45:13] [Champion] 🏆 NEW BEST! Trial 5: mean_reward=75.0000 params={'n_steer': 8} [2026-04-17 14:45:13] [AutoResearch] Only 1 results — using random proposal. +[2026-04-17 22:10:12] [AutoResearch] GP UCB top-5 candidates: +[2026-04-17 22:10:12] UCB=2.3107 mu=0.3981 sigma=0.9563 params={'n_steer': 9, 'n_throttle': 2, 'learning_rate': 0.001405531880392808, 'timesteps': 26173} +[2026-04-17 22:10:12] UCB=2.3049 mu=0.8602 sigma=0.7224 params={'n_steer': 9, 'n_throttle': 3, 'learning_rate': 0.001793493447174312, 'timesteps': 19198} +[2026-04-17 22:10:12] UCB=2.2813 mu=0.4904 sigma=0.8954 params={'n_steer': 9, 'n_throttle': 4, 'learning_rate': 0.0011616192816742616, 'timesteps': 13887} +[2026-04-17 22:10:12] UCB=2.2767 mu=0.5194 sigma=0.8787 params={'n_steer': 9, 'n_throttle': 4, 'learning_rate': 0.0011646447444663046, 'timesteps': 21199} +[2026-04-17 22:10:12] UCB=2.2525 mu=0.6254 sigma=0.8136 params={'n_steer': 9, 'n_throttle': 3, 'learning_rate': 0.0010196345864901517, 'timesteps': 22035} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 1: mean_reward=50.0000 params={'n_steer': 5} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 1: mean_reward=80.0000 params={'n_steer': 7} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 0: mean_reward=50.0000 params={'r': 50} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 1: mean_reward=80.0000 params={'r': 80} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 3: mean_reward=90.0000 params={'r': 90} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 5: mean_reward=75.0000 params={'n_steer': 8} +[2026-04-17 22:10:12] [AutoResearch] Only 1 results — using random proposal. diff --git a/agent/outerloop-results/autoresearch_phase3_log.txt b/agent/outerloop-results/autoresearch_phase3_log.txt index 1ea86fb..9b611bd 100644 --- a/agent/outerloop-results/autoresearch_phase3_log.txt +++ b/agent/outerloop-results/autoresearch_phase3_log.txt @@ -400,3 +400,8 @@ [2026-04-17 14:45:26] [Wave3] Only 0 results — using random proposal. [2026-04-17 14:45:26] [Champion] 🏆 NEW BEST! Trial 3: score=1500.00 (mini_monaco=1500.0) params={'learning_rate': 0.0002, 'steps_per_switch': 8000, 'total_timesteps': 150000} [2026-04-17 14:45:26] [Champion] 🏆 NEW BEST! Trial 1: score=2000.00 (mini_monaco=2000.0) params={} +[2026-04-17 22:10:26] [Wave3] Seed trial 1/2: using hardcoded params. +[2026-04-17 22:10:26] [Wave3] Seed trial 2/2: using hardcoded params. +[2026-04-17 22:10:26] [Wave3] Only 0 results — using random proposal. +[2026-04-17 22:10:26] [Champion] 🏆 NEW BEST! Trial 3: score=1500.00 (mini_monaco=1500.0) params={'learning_rate': 0.0002, 'steps_per_switch': 8000, 'total_timesteps': 150000} +[2026-04-17 22:10:26] [Champion] 🏆 NEW BEST! Trial 1: score=2000.00 (mini_monaco=2000.0) params={}