fix: exp19 — hard episode time limit to stop minutes-long stuck cars

StuckTerminationWrapper wall-clock timer was resettable by barrier-sliding:
car drifting 0.5m along a wall repeatedly resets the 12s timer. At low sim
fps (1-2fps when both cars stuck), 40-step check also takes minutes.

Fix: added max_episode_seconds=30 — hard wall-clock limit per episode,
independent of position or sim fps. No episode can run longer than 30s.

Also adds monitor_training.sh: independent shell process that checks every
5 minutes and appends status to /tmp/training_monitor.log — works without
Claude being active.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

agent/experiments/exp19_parallel_450k_v3.py

@@ -0,0 +1,200 @@
+"""
+Exp 19: Parallel DummyVecEnv — 450k steps, fixed stuck detection (v3).
+
+Fixes from Exp 18:
+  - Exp 17 fix carried forward: window_size=200, min_lap_time=12s
+
+New fix for Exp 19:
+  StuckTerminationWrapper wall-clock timer was resettable: a car slowly
+  sliding along a barrier can drift 0.5m repeatedly, resetting the 12s
+  timer indefinitely. At low sim fps (1-2fps when both cars stuck against
+  walls), even the 40-step check takes minutes. Cars were stuck visually
+  for minutes with no episode termination.
+
+  Fix: hard max_episode_seconds=30 in StuckTerminationWrapper. Every
+  episode terminates after 30 wall-clock seconds regardless of car
+  position, sim fps, or barrier sliding. No episode can stall longer.
+
+Setup — TWO sim instances required:
+  Sim 1: donkey_sim.exe on port 9091 → generated_track
+  Sim 2: separate copy of donkey_sim.exe on port 9093 → mountain_track
+"""
+import sys, os, time
+sys.path.insert(0, '/home/paulh/projects/donkeycar-rl-autoresearch/agent')
+
+from multitrack_runner import log, StuckTerminationWrapper
+from donkeycar_sb3_runner import ThrottleClampWrapper
+from reward_wrapper import SpeedRewardWrapper
+from stable_baselines3 import PPO
+from stable_baselines3.common.vec_env import DummyVecEnv, VecTransposeImage
+import gymnasium as gym
+import numpy as np
+
+HOST             = 'localhost'
+THROTTLE_MIN     = 0.2
+LR               = 0.000725
+TOTAL_STEPS      = 450_000
+CHECKPOINT_EVERY = 20_000
+SAVE_DIR         = '/home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp19-parallel-450k-v3'
+os.makedirs(SAVE_DIR, exist_ok=True)
+
+# Exploit fixes: larger window catches full circles; higher min_lap_time
+# kills sub-genuine laps before they can contribute positive reward.
+EFFICIENCY_WINDOW = 200   # was 30 — now covers 2+ exploit circles at 22fps
+MIN_LAP_TIME      = 12.0  # was 5.0 — genuine laps: gentrack 13-16s, mountain 27-29s
+
+
+def make_env(track_id, port):
+    def _init():
+        raw = gym.make(track_id, conf={'host': HOST, 'port': port})
+        env = ThrottleClampWrapper(raw, throttle_min=THROTTLE_MIN)
+        env = StuckTerminationWrapper(env, stuck_steps=40, min_displacement=0.5,
+                                      max_episode_seconds=30.0)
+        env = SpeedRewardWrapper(env, window_size=EFFICIENCY_WINDOW, min_lap_time=MIN_LAP_TIME)
+        return env
+    return _init
+
+
+log('=' * 60)
+log('Exp 19: Parallel DummyVecEnv — 450k steps (stuck fix v3)')
+log(f'  Sim 1: {HOST}:9091 → generated_track')
+log(f'  Sim 2: {HOST}:9093 → mountain_track')
+log(f'  throttle_min={THROTTLE_MIN}, lr={LR}, total={TOTAL_STEPS:,}')
+log(f'  Reward: v6 + exploit fix (window={EFFICIENCY_WINDOW}, min_lap={MIN_LAP_TIME}s)')
+log(f'  Stuck termination: 40 steps (~2.5s)')
+log(f'  Checkpoints: every {CHECKPOINT_EVERY:,} steps')
+log('=' * 60)
+
+log('Creating DummyVecEnv with two tracks...')
+env = DummyVecEnv([
+    make_env('donkey-generated-track-v0', 9091),
+    make_env('donkey-mountain-track-v0', 9093),
+])
+env = VecTransposeImage(env)
+log(f'  VecEnv num_envs={env.num_envs}, obs={env.observation_space.shape}')
+
+model = PPO('CnnPolicy', env, learning_rate=LR, verbose=1, device='cpu')
+log('PPO created. Starting training...')
+
+best_reward = float('-inf')
+steps_done = 0
+
+while steps_done < TOTAL_STEPS:
+    seg_steps = min(CHECKPOINT_EVERY, TOTAL_STEPS - steps_done)
+    model.learn(total_timesteps=seg_steps, reset_num_timesteps=False)
+    steps_done += seg_steps
+
+    ckpt = os.path.join(SAVE_DIR, f'checkpoint_{steps_done:07d}')
+    model.save(ckpt)
+    model.save(os.path.join(SAVE_DIR, 'model'))
+    log(f'[{steps_done:,}/{TOTAL_STEPS:,}] Checkpoint saved: {ckpt}.zip')
+
+    # Eval on both training tracks using the existing DummyVecEnv connections
+    try:
+        obs = env.reset()
+        ep_rewards = np.zeros(env.num_envs)
+        ep_steps   = np.zeros(env.num_envs)
+        done_mask  = np.zeros(env.num_envs, dtype=bool)
+        for _ in range(2000):
+            action, _ = model.predict(obs, deterministic=True)
+            obs, rewards, dones, infos = env.step(action)
+            for i in range(env.num_envs):
+                if not done_mask[i]:
+                    ep_rewards[i] += rewards[i]
+                    ep_steps[i]   += 1
+                    if dones[i]:
+                        done_mask[i] = True
+            if done_mask.all():
+                break
+
+        status0 = '✅' if ep_steps[0] >= 2000 else f'❌@{int(ep_steps[0])}'
+        status1 = '✅' if ep_steps[1] >= 2000 else f'❌@{int(ep_steps[1])}'
+        log(f'  Eval: gen_track={ep_rewards[0]:.1f}r/{int(ep_steps[0])}s {status0}  '
+            f'mountain={ep_rewards[1]:.1f}r/{int(ep_steps[1])}s {status1}')
+
+        total_reward = ep_rewards.sum()
+        if total_reward > best_reward:
+            best_reward = total_reward
+            model.save(os.path.join(SAVE_DIR, 'best_model'))
+            log(f'  ⭐ NEW BEST: {best_reward:.1f} combined reward')
+    except Exception as e:
+        log(f'  Eval error: {e}')
+        import traceback; traceback.print_exc()
+
+model.save(os.path.join(SAVE_DIR, 'model'))
+log(f'\nTraining complete. Best combined reward: {best_reward:.1f}')
+
+env.close()
+time.sleep(5)
+
+# --- Final eval on all 4 tracks (sequential, port 9091) ---
+log('\n' + '=' * 60)
+log('FINAL EVALUATION: best_model on 4 tracks (3 sets each)')
+log('=' * 60)
+
+EVAL_TRACKS = [
+    ('donkey-generated-track-v0',  'generated_track'),
+    ('donkey-mountain-track-v0',   'mountain_track'),
+    ('donkey-minimonaco-track-v0', 'mini_monaco'),
+    ('donkey-generated-roads-v0',  'generated_road'),
+]
+EVAL_PORT     = 9091
+EVAL_SETS     = 3
+EVAL_MAX_STEPS = 2000
+
+best_model_path  = os.path.join(SAVE_DIR, 'best_model.zip')
+results_by_track = {}
+
+for track_id, track_name in EVAL_TRACKS:
+    log(f'\n--- {track_name} ---')
+    steps_list = []
+
+    for s in range(1, EVAL_SETS + 1):
+        try:
+            raw       = gym.make(track_id, conf={'host': HOST, 'port': EVAL_PORT})
+            inner     = ThrottleClampWrapper(raw, throttle_min=THROTTLE_MIN)
+            inner     = StuckTerminationWrapper(inner, stuck_steps=40, min_displacement=0.5)
+            inner     = SpeedRewardWrapper(inner)
+            eval_env  = VecTransposeImage(DummyVecEnv([lambda e=inner: e]))
+
+            eval_model = PPO.load(best_model_path, env=eval_env, device='cpu')
+
+            obs = eval_env.reset()
+            total_r, total_s, done = 0.0, 0, False
+            while not done and total_s < EVAL_MAX_STEPS:
+                action, _ = eval_model.predict(obs, deterministic=True)
+                result = eval_env.step(action)
+                if len(result) == 4:
+                    obs, r, d, info = result
+                    done = bool(d[0])
+                else:
+                    obs, r, t, tr, info = result
+                    done = bool(t[0] or tr[0])
+                total_r += float(r[0])
+                total_s += 1
+
+            status = '✅' if total_s >= EVAL_MAX_STEPS else f'❌@{total_s}'
+            log(f'  Set {s}: {total_r:.1f}r / {total_s}s {status}')
+            steps_list.append(total_s)
+
+            eval_env.close()
+            time.sleep(3)
+        except Exception as e:
+            log(f'  Set {s}: ERROR — {e}')
+            steps_list.append(0)
+            time.sleep(3)
+
+    mean_steps = np.mean(steps_list) if steps_list else 0
+    results_by_track[track_name] = steps_list
+    log(f'  Mean: {mean_steps:.0f} steps')
+
+log('\n' + '=' * 60)
+log('SUMMARY')
+log('=' * 60)
+for track_name, steps_list in results_by_track.items():
+    steps_str = '/'.join(str(s) for s in steps_list)
+    mean      = np.mean(steps_list)
+    verdict   = '✅' if mean >= 1500 else '⚠️' if mean >= 500 else '❌'
+    log(f'  {verdict} {track_name:20s}: {steps_str}  mean={mean:.0f}')
+
+log(f'\n=== Exp 17 COMPLETE ===')

agent/multitrack_runner.py

@@ -143,19 +143,22 @@ class StuckTerminationWrapper(gym.Wrapper):
     When stuck is detected: terminated=True so SpeedRewardWrapper returns -1.0.
     """
     def __init__(self, env, stuck_steps: int = 80, min_displacement: float = 0.5,
-                 max_stuck_seconds: float = 12.0):
+                 max_stuck_seconds: float = 12.0, max_episode_seconds: float = 30.0):
         super().__init__(env)
-        self.stuck_steps        = stuck_steps
+        self.stuck_steps          = stuck_steps
-        self.min_displacement   = min_displacement
+        self.min_displacement     = min_displacement
-        self.max_stuck_seconds  = max_stuck_seconds
+        self.max_stuck_seconds    = max_stuck_seconds
-        self._pos_buf: deque    = deque(maxlen=stuck_steps)
+        self.max_episode_seconds  = max_episode_seconds
-        self._last_progress_pos = None
+        self._pos_buf: deque      = deque(maxlen=stuck_steps)
-        self._last_progress_t   = None
+        self._last_progress_pos   = None
+        self._last_progress_t     = None
+        self._episode_start_t     = None
 
     def reset(self, **kwargs):
         self._pos_buf.clear()
         self._last_progress_pos = None
         self._last_progress_t   = None
+        self._episode_start_t   = time.time()
         return self.env.reset(**kwargs)
 
     def step(self, action):
@@ -194,6 +197,15 @@ class StuckTerminationWrapper(gym.Wrapper):
             except (TypeError, ValueError):
                 pass
 
+        # Hard episode wall-clock limit — fires regardless of car position or sim fps.
+        # Catches cars sliding slowly along barriers that keep resetting the
+        # max_stuck_seconds timer by drifting 0.5m at a time.
+        if not terminated and self._episode_start_t is not None:
+            if (now - self._episode_start_t) > self.max_episode_seconds:
+                terminated = True
+                info['stuck_termination'] = True
+                info['stuck_reason'] = 'episode_timeout'
+
         # Step-count stuck detection (original logic)
         if not terminated and len(self._pos_buf) >= self.stuck_steps:
             displacement = float(np.linalg.norm(

monitor_training.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# Standalone training monitor — runs independently of Claude.
+# Usage: bash monitor_training.sh <log_file> <pid>
+# Output: appended to /tmp/training_monitor.log
+#
+# Checks every 5 minutes:
+#   - Is the training process still alive?
+#   - What are the most recent checkpoint eval scores?
+#   - Are there any errors or exploit laps?
+#   - What is the current step count?
+
+LOG_FILE="${1:-/tmp/exp19.log}"
+TRAIN_PID="${2:-}"
+MONITOR_OUT="/tmp/training_monitor.log"
+INTERVAL=300  # 5 minutes
+
+echo "======================================" >> "$MONITOR_OUT"
+echo "Monitor started: $(date)" >> "$MONITOR_OUT"
+echo "Watching: $LOG_FILE  PID: $TRAIN_PID" >> "$MONITOR_OUT"
+echo "======================================" >> "$MONITOR_OUT"
+
+while true; do
+    sleep "$INTERVAL"
+    echo "" >> "$MONITOR_OUT"
+    echo "--- $(date) ---" >> "$MONITOR_OUT"
+
+    # Check process alive
+    if [ -n "$TRAIN_PID" ]; then
+        if ps -p "$TRAIN_PID" > /dev/null 2>&1; then
+            echo "Process $TRAIN_PID: RUNNING" >> "$MONITOR_OUT"
+        else
+            echo "Process $TRAIN_PID: STOPPED" >> "$MONITOR_OUT"
+            echo "Training ended at $(date)" >> "$MONITOR_OUT"
+            break
+        fi
+    fi
+
+    # Latest checkpoint/eval/best lines
+    echo "Recent checkpoints:" >> "$MONITOR_OUT"
+    grep "Checkpoint\|Eval:\|NEW BEST" "$LOG_FILE" 2>/dev/null | tail -6 >> "$MONITOR_OUT"
+
+    # Step progress
+    echo "Step progress:" >> "$MONITOR_OUT"
+    grep "total_timesteps" "$LOG_FILE" 2>/dev/null | tail -1 >> "$MONITOR_OUT"
+
+    # Exploit warning: more than 5 lap times in the last 100 lines
+    LAP_COUNT=$(tail -100 "$LOG_FILE" 2>/dev/null | grep -c "New lap time")
+    echo "Laps in last 100 log lines: $LAP_COUNT" >> "$MONITOR_OUT"
+    if [ "$LAP_COUNT" -gt 10 ]; then
+        echo "WARNING: high lap count may indicate circular exploit" >> "$MONITOR_OUT"
+    fi
+
+    # Any errors
+    ERRORS=$(grep -c "ERROR\|Traceback\|Exception" "$LOG_FILE" 2>/dev/null)
+    if [ "$ERRORS" -gt 0 ]; then
+        echo "ERRORS DETECTED: $ERRORS error lines in log" >> "$MONITOR_OUT"
+        grep "ERROR\|Traceback" "$LOG_FILE" 2>/dev/null | tail -3 >> "$MONITOR_OUT"
+    fi
+done
+
+echo "Monitor exiting: $(date)" >> "$MONITOR_OUT"