From ca748c504ad0658c236cc2835f9b50d54db55d3e Mon Sep 17 00:00:00 2001 From: Paul Huliganga Date: Thu, 14 May 2026 15:32:59 -0400 Subject: [PATCH] chore(models): add exp24/exp25 training logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exp24 (discrete steering, road regen): 200k steps, 19 consecutive full episodes, best checkpoint 170k → 365.5r/2000s. Exp25 (wheel OverlapSphere fix): extends exp24 with any-angle barrier detection; multiple restart logs from debugging session. Co-Authored-By: Claude Sonnet 4.6 --- agent/models/exp24-discrete/current.pid | 1 + .../run_2026-05-05_200903_discrete.log | 0 agent/models/exp25-wheel-fix/current.pid | 1 + .../run_2026-05-05_221255_wheel_fix.log | 14 +++ .../run_2026-05-05_223230_wheel_fix.log | 4 + .../run_2026-05-05_223353_wheel_fix.log | 100 ++++++++++++++++++ .../run_2026-05-06_112832_wheel_fix.log | 13 +++ 7 files changed, 133 insertions(+) create mode 100644 agent/models/exp24-discrete/current.pid create mode 100644 agent/models/exp24-discrete/run_2026-05-05_200903_discrete.log create mode 100644 agent/models/exp25-wheel-fix/current.pid create mode 100644 agent/models/exp25-wheel-fix/run_2026-05-05_221255_wheel_fix.log create mode 100644 agent/models/exp25-wheel-fix/run_2026-05-05_223230_wheel_fix.log create mode 100644 agent/models/exp25-wheel-fix/run_2026-05-05_223353_wheel_fix.log create mode 100644 agent/models/exp25-wheel-fix/run_2026-05-06_112832_wheel_fix.log diff --git a/agent/models/exp24-discrete/current.pid b/agent/models/exp24-discrete/current.pid new file mode 100644 index 0000000..d260379 --- /dev/null +++ b/agent/models/exp24-discrete/current.pid @@ -0,0 +1 @@ +733053 \ No newline at end of file diff --git a/agent/models/exp24-discrete/run_2026-05-05_200903_discrete.log b/agent/models/exp24-discrete/run_2026-05-05_200903_discrete.log new file mode 100644 index 0000000..e69de29 diff --git a/agent/models/exp25-wheel-fix/current.pid b/agent/models/exp25-wheel-fix/current.pid new file mode 100644 index 0000000..a56d72f --- /dev/null +++ b/agent/models/exp25-wheel-fix/current.pid @@ -0,0 +1 @@ +1037851 \ No newline at end of file diff --git a/agent/models/exp25-wheel-fix/run_2026-05-05_221255_wheel_fix.log b/agent/models/exp25-wheel-fix/run_2026-05-05_221255_wheel_fix.log new file mode 100644 index 0000000..84af937 --- /dev/null +++ b/agent/models/exp25-wheel-fix/run_2026-05-05_221255_wheel_fix.log @@ -0,0 +1,14 @@ +[22:12:55] ============================================================ +[22:12:55] Exp 25 started — PID 776352 +[22:12:55] Log: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/run_2026-05-05_221255_wheel_fix.log +[22:12:55] ============================================================ +[22:19:52] [10,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0010000.zip +[22:19:52] Reconnecting for fresh road... +[22:19:57] Connected (new road) +[22:20:23] Eval: gen_road=312.7r/2000s ✅ +[22:20:24] NEW BEST: steps=2000 reward=312.7 +[22:26:19] [20,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0020000.zip +[22:26:19] Reconnecting for fresh road... +[22:26:24] Connected (new road) +[22:26:52] Eval: gen_road=320.1r/2000s ✅ +[22:26:53] NEW BEST: steps=2000 reward=320.1 diff --git a/agent/models/exp25-wheel-fix/run_2026-05-05_223230_wheel_fix.log b/agent/models/exp25-wheel-fix/run_2026-05-05_223230_wheel_fix.log new file mode 100644 index 0000000..caa3a45 --- /dev/null +++ b/agent/models/exp25-wheel-fix/run_2026-05-05_223230_wheel_fix.log @@ -0,0 +1,4 @@ +[22:32:30] ============================================================ +[22:32:30] Exp 25 started — PID 782953 +[22:32:30] Log: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/run_2026-05-05_223230_wheel_fix.log +[22:32:30] ============================================================ diff --git a/agent/models/exp25-wheel-fix/run_2026-05-05_223353_wheel_fix.log b/agent/models/exp25-wheel-fix/run_2026-05-05_223353_wheel_fix.log new file mode 100644 index 0000000..e82dce3 --- /dev/null +++ b/agent/models/exp25-wheel-fix/run_2026-05-05_223353_wheel_fix.log @@ -0,0 +1,100 @@ +[22:33:53] ============================================================ +[22:33:53] Exp 25 started — PID 783482 +[22:33:53] Log: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/run_2026-05-05_223353_wheel_fix.log +[22:33:53] ============================================================ +[22:40:48] [10,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0010000.zip +[22:40:48] Reconnecting for fresh road... +[22:40:53] Connected (new road) +[22:41:02] Eval: gen_road=74.2r/566s ❌@566 +[22:41:03] NEW BEST: steps=566 reward=74.2 +[22:46:30] [20,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0020000.zip +[22:46:30] Reconnecting for fresh road... +[22:46:35] Connected (new road) +[22:47:01] Eval: gen_road=240.0r/2000s ✅ +[22:47:01] NEW BEST: steps=2000 reward=240.0 +[22:52:06] [30,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0030000.zip +[22:52:06] Reconnecting for fresh road... +[22:52:11] Connected (new road) +[22:52:37] Eval: gen_road=260.4r/2000s ✅ +[22:52:37] NEW BEST: steps=2000 reward=260.4 +[22:57:31] [40,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0040000.zip +[22:57:31] Reconnecting for fresh road... +[22:57:36] Connected (new road) +[22:58:03] Eval: gen_road=323.8r/2000s ✅ +[22:58:03] NEW BEST: steps=2000 reward=323.8 +[23:03:07] [50,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0050000.zip +[23:03:07] Reconnecting for fresh road... +[23:03:13] Connected (new road) +[23:03:39] Eval: gen_road=352.8r/2000s ✅ +[23:03:39] NEW BEST: steps=2000 reward=352.8 +[23:08:56] [60,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0060000.zip +[23:08:56] Reconnecting for fresh road... +[23:09:01] Connected (new road) +[23:09:27] Eval: gen_road=362.7r/2000s ✅ +[23:09:28] NEW BEST: steps=2000 reward=362.7 +[23:14:31] [70,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0070000.zip +[23:14:31] Reconnecting for fresh road... +[23:14:37] Connected (new road) +[23:15:03] Eval: gen_road=365.0r/2000s ✅ +[23:15:03] NEW BEST: steps=2000 reward=365.0 +[23:20:08] [80,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0080000.zip +[23:20:08] Reconnecting for fresh road... +[23:20:13] Connected (new road) +[23:20:39] Eval: gen_road=381.0r/2000s ✅ +[23:20:39] NEW BEST: steps=2000 reward=381.0 +[23:25:45] [90,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0090000.zip +[23:25:45] Reconnecting for fresh road... +[23:25:50] Connected (new road) +[23:26:17] Eval: gen_road=344.7r/2000s ✅ +[23:31:47] [100,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0100000.zip +[23:31:47] Reconnecting for fresh road... +[23:31:52] Connected (new road) +[23:32:18] Eval: gen_road=362.9r/2000s ✅ +[23:37:44] [110,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0110000.zip +[23:37:44] Reconnecting for fresh road... +[23:37:49] Connected (new road) +[23:38:15] Eval: gen_road=345.0r/2000s ✅ +[23:43:11] [120,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0120000.zip +[23:43:11] Reconnecting for fresh road... +[23:43:16] Connected (new road) +[23:43:27] Eval: gen_road=114.8r/709s ❌@709 +[23:48:20] [130,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0130000.zip +[23:48:20] Reconnecting for fresh road... +[23:48:25] Connected (new road) +[23:48:52] Eval: gen_road=322.1r/2000s ✅ +[23:53:45] [140,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0140000.zip +[23:53:45] Reconnecting for fresh road... +[23:53:50] Connected (new road) +[23:54:17] Eval: gen_road=281.5r/1935s ❌@1935 +[23:59:18] [150,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0150000.zip +[23:59:18] Reconnecting for fresh road... +[23:59:23] Connected (new road) +[23:59:49] Eval: gen_road=299.4r/2000s ✅ +[00:05:05] [160,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0160000.zip +[00:05:05] Reconnecting for fresh road... +[00:05:10] Connected (new road) +[00:05:36] Eval: gen_road=292.5r/2000s ✅ +[00:11:09] [170,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0170000.zip +[00:11:09] Reconnecting for fresh road... +[00:11:14] Connected (new road) +[00:11:41] Eval: gen_road=297.8r/2000s ✅ +[00:17:06] [180,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0180000.zip +[00:17:06] Reconnecting for fresh road... +[00:17:11] Connected (new road) +[00:17:38] Eval: gen_road=314.6r/2000s ✅ +[00:23:07] [190,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0190000.zip +[00:23:07] Reconnecting for fresh road... +[00:23:12] Connected (new road) +[00:23:38] Eval: gen_road=322.6r/2000s ✅ +[00:29:03] [200,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0200000.zip +[00:29:03] Reconnecting for fresh road... +[00:29:08] Connected (new road) +[00:29:35] Eval: gen_road=313.6r/2000s ✅ +[00:29:35] ============================================================ +[00:29:35] FINAL EVALUATION: best_model on generated_road (3 fresh roads) +[00:29:35] ============================================================ +[00:30:07] Set 1: 383.4r / 2000s ✅ +[00:30:40] Set 2: 378.7r / 2000s ✅ +[00:31:10] Set 3: 334.7r / 1776s ❌@1776 +[00:31:10] Mean: 1925 steps / 365.6 reward +[00:31:10] Exp 25 complete. diff --git a/agent/models/exp25-wheel-fix/run_2026-05-06_112832_wheel_fix.log b/agent/models/exp25-wheel-fix/run_2026-05-06_112832_wheel_fix.log new file mode 100644 index 0000000..37bec2e --- /dev/null +++ b/agent/models/exp25-wheel-fix/run_2026-05-06_112832_wheel_fix.log @@ -0,0 +1,13 @@ +[11:28:32] ============================================================ +[11:28:32] Exp 25 started — PID 1037851 +[11:28:32] Log: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/run_2026-05-06_112832_wheel_fix.log +[11:28:32] ============================================================ +[11:36:36] [10,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0010000.zip +[11:36:36] Reconnecting for fresh road... +[11:36:41] Connected (new road) +[11:36:45] Eval: gen_road=5.6r/83s ❌@83 +[11:36:45] NEW BEST: steps=83 reward=5.6 +[11:44:53] [20,000/200,000] Checkpoint saved: /home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp25-wheel-fix/checkpoint_0020000.zip +[11:44:53] Reconnecting for fresh road... +[11:44:58] Connected (new road) +[11:45:01] Eval: gen_road=3.1r/70s ❌@70