diff --git a/agent/multitrack_runner.py b/agent/multitrack_runner.py index 49d6d3a..b449662 100644 --- a/agent/multitrack_runner.py +++ b/agent/multitrack_runner.py @@ -335,8 +335,13 @@ def train_multitrack(model, first_env, total_timesteps, steps_per_switch, if save_dir: try: os.makedirs(save_dir, exist_ok=True) + # Save numbered checkpoint — NEVER overwrite previous checkpoints. + # Every segment is preserved so we can return to any point in training. + ckpt_name = f'checkpoint_{steps_done:07d}' + model.save(os.path.join(save_dir, ckpt_name)) + # Also overwrite 'model' for crash-recovery (latest weights) model.save(os.path.join(save_dir, 'model')) - log(f'[W3 Runner] Checkpoint saved at step {steps_done:,}') + log(f'[W3 Runner] Checkpoint saved: {ckpt_name} (step {steps_done:,})') except Exception as e: log(f'[W3 Runner] WARNING: checkpoint save failed: {e}') diff --git a/agent/outerloop-results/autoresearch_phase2_log.txt b/agent/outerloop-results/autoresearch_phase2_log.txt index 9c76714..ae9fc81 100644 --- a/agent/outerloop-results/autoresearch_phase2_log.txt +++ b/agent/outerloop-results/autoresearch_phase2_log.txt @@ -749,3 +749,16 @@ [2026-04-17 14:45:13] [Champion] 🏆 NEW BEST! Trial 3: mean_reward=90.0000 params={'r': 90} [2026-04-17 14:45:13] [Champion] 🏆 NEW BEST! Trial 5: mean_reward=75.0000 params={'n_steer': 8} [2026-04-17 14:45:13] [AutoResearch] Only 1 results — using random proposal. +[2026-04-17 22:10:12] [AutoResearch] GP UCB top-5 candidates: +[2026-04-17 22:10:12] UCB=2.3107 mu=0.3981 sigma=0.9563 params={'n_steer': 9, 'n_throttle': 2, 'learning_rate': 0.001405531880392808, 'timesteps': 26173} +[2026-04-17 22:10:12] UCB=2.3049 mu=0.8602 sigma=0.7224 params={'n_steer': 9, 'n_throttle': 3, 'learning_rate': 0.001793493447174312, 'timesteps': 19198} +[2026-04-17 22:10:12] UCB=2.2813 mu=0.4904 sigma=0.8954 params={'n_steer': 9, 'n_throttle': 4, 'learning_rate': 0.0011616192816742616, 'timesteps': 13887} +[2026-04-17 22:10:12] UCB=2.2767 mu=0.5194 sigma=0.8787 params={'n_steer': 9, 'n_throttle': 4, 'learning_rate': 0.0011646447444663046, 'timesteps': 21199} +[2026-04-17 22:10:12] UCB=2.2525 mu=0.6254 sigma=0.8136 params={'n_steer': 9, 'n_throttle': 3, 'learning_rate': 0.0010196345864901517, 'timesteps': 22035} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 1: mean_reward=50.0000 params={'n_steer': 5} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 1: mean_reward=80.0000 params={'n_steer': 7} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 0: mean_reward=50.0000 params={'r': 50} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 1: mean_reward=80.0000 params={'r': 80} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 3: mean_reward=90.0000 params={'r': 90} +[2026-04-17 22:10:12] [Champion] 🏆 NEW BEST! Trial 5: mean_reward=75.0000 params={'n_steer': 8} +[2026-04-17 22:10:12] [AutoResearch] Only 1 results — using random proposal. diff --git a/agent/outerloop-results/autoresearch_phase3_log.txt b/agent/outerloop-results/autoresearch_phase3_log.txt index 1ea86fb..9b611bd 100644 --- a/agent/outerloop-results/autoresearch_phase3_log.txt +++ b/agent/outerloop-results/autoresearch_phase3_log.txt @@ -400,3 +400,8 @@ [2026-04-17 14:45:26] [Wave3] Only 0 results — using random proposal. [2026-04-17 14:45:26] [Champion] 🏆 NEW BEST! Trial 3: score=1500.00 (mini_monaco=1500.0) params={'learning_rate': 0.0002, 'steps_per_switch': 8000, 'total_timesteps': 150000} [2026-04-17 14:45:26] [Champion] 🏆 NEW BEST! Trial 1: score=2000.00 (mini_monaco=2000.0) params={} +[2026-04-17 22:10:26] [Wave3] Seed trial 1/2: using hardcoded params. +[2026-04-17 22:10:26] [Wave3] Seed trial 2/2: using hardcoded params. +[2026-04-17 22:10:26] [Wave3] Only 0 results — using random proposal. +[2026-04-17 22:10:26] [Champion] 🏆 NEW BEST! Trial 3: score=1500.00 (mini_monaco=1500.0) params={'learning_rate': 0.0002, 'steps_per_switch': 8000, 'total_timesteps': 150000} +[2026-04-17 22:10:26] [Champion] 🏆 NEW BEST! Trial 1: score=2000.00 (mini_monaco=2000.0) params={}