diff --git a/agent/experiments/exp19_parallel_450k_v3.py b/agent/experiments/exp19_parallel_450k_v3.py new file mode 100644 index 0000000..009f5df --- /dev/null +++ b/agent/experiments/exp19_parallel_450k_v3.py @@ -0,0 +1,200 @@ +""" +Exp 19: Parallel DummyVecEnv — 450k steps, fixed stuck detection (v3). + +Fixes from Exp 18: + - Exp 17 fix carried forward: window_size=200, min_lap_time=12s + +New fix for Exp 19: + StuckTerminationWrapper wall-clock timer was resettable: a car slowly + sliding along a barrier can drift 0.5m repeatedly, resetting the 12s + timer indefinitely. At low sim fps (1-2fps when both cars stuck against + walls), even the 40-step check takes minutes. Cars were stuck visually + for minutes with no episode termination. + + Fix: hard max_episode_seconds=30 in StuckTerminationWrapper. Every + episode terminates after 30 wall-clock seconds regardless of car + position, sim fps, or barrier sliding. No episode can stall longer. + +Setup — TWO sim instances required: + Sim 1: donkey_sim.exe on port 9091 → generated_track + Sim 2: separate copy of donkey_sim.exe on port 9093 → mountain_track +""" +import sys, os, time +sys.path.insert(0, '/home/paulh/projects/donkeycar-rl-autoresearch/agent') + +from multitrack_runner import log, StuckTerminationWrapper +from donkeycar_sb3_runner import ThrottleClampWrapper +from reward_wrapper import SpeedRewardWrapper +from stable_baselines3 import PPO +from stable_baselines3.common.vec_env import DummyVecEnv, VecTransposeImage +import gymnasium as gym +import numpy as np + +HOST = 'localhost' +THROTTLE_MIN = 0.2 +LR = 0.000725 +TOTAL_STEPS = 450_000 +CHECKPOINT_EVERY = 20_000 +SAVE_DIR = '/home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp19-parallel-450k-v3' +os.makedirs(SAVE_DIR, exist_ok=True) + +# Exploit fixes: larger window catches full circles; higher min_lap_time +# kills sub-genuine laps before they can contribute positive reward. +EFFICIENCY_WINDOW = 200 # was 30 — now covers 2+ exploit circles at 22fps +MIN_LAP_TIME = 12.0 # was 5.0 — genuine laps: gentrack 13-16s, mountain 27-29s + + +def make_env(track_id, port): + def _init(): + raw = gym.make(track_id, conf={'host': HOST, 'port': port}) + env = ThrottleClampWrapper(raw, throttle_min=THROTTLE_MIN) + env = StuckTerminationWrapper(env, stuck_steps=40, min_displacement=0.5, + max_episode_seconds=30.0) + env = SpeedRewardWrapper(env, window_size=EFFICIENCY_WINDOW, min_lap_time=MIN_LAP_TIME) + return env + return _init + + +log('=' * 60) +log('Exp 19: Parallel DummyVecEnv — 450k steps (stuck fix v3)') +log(f' Sim 1: {HOST}:9091 → generated_track') +log(f' Sim 2: {HOST}:9093 → mountain_track') +log(f' throttle_min={THROTTLE_MIN}, lr={LR}, total={TOTAL_STEPS:,}') +log(f' Reward: v6 + exploit fix (window={EFFICIENCY_WINDOW}, min_lap={MIN_LAP_TIME}s)') +log(f' Stuck termination: 40 steps (~2.5s)') +log(f' Checkpoints: every {CHECKPOINT_EVERY:,} steps') +log('=' * 60) + +log('Creating DummyVecEnv with two tracks...') +env = DummyVecEnv([ + make_env('donkey-generated-track-v0', 9091), + make_env('donkey-mountain-track-v0', 9093), +]) +env = VecTransposeImage(env) +log(f' VecEnv num_envs={env.num_envs}, obs={env.observation_space.shape}') + +model = PPO('CnnPolicy', env, learning_rate=LR, verbose=1, device='cpu') +log('PPO created. Starting training...') + +best_reward = float('-inf') +steps_done = 0 + +while steps_done < TOTAL_STEPS: + seg_steps = min(CHECKPOINT_EVERY, TOTAL_STEPS - steps_done) + model.learn(total_timesteps=seg_steps, reset_num_timesteps=False) + steps_done += seg_steps + + ckpt = os.path.join(SAVE_DIR, f'checkpoint_{steps_done:07d}') + model.save(ckpt) + model.save(os.path.join(SAVE_DIR, 'model')) + log(f'[{steps_done:,}/{TOTAL_STEPS:,}] Checkpoint saved: {ckpt}.zip') + + # Eval on both training tracks using the existing DummyVecEnv connections + try: + obs = env.reset() + ep_rewards = np.zeros(env.num_envs) + ep_steps = np.zeros(env.num_envs) + done_mask = np.zeros(env.num_envs, dtype=bool) + for _ in range(2000): + action, _ = model.predict(obs, deterministic=True) + obs, rewards, dones, infos = env.step(action) + for i in range(env.num_envs): + if not done_mask[i]: + ep_rewards[i] += rewards[i] + ep_steps[i] += 1 + if dones[i]: + done_mask[i] = True + if done_mask.all(): + break + + status0 = '✅' if ep_steps[0] >= 2000 else f'❌@{int(ep_steps[0])}' + status1 = '✅' if ep_steps[1] >= 2000 else f'❌@{int(ep_steps[1])}' + log(f' Eval: gen_track={ep_rewards[0]:.1f}r/{int(ep_steps[0])}s {status0} ' + f'mountain={ep_rewards[1]:.1f}r/{int(ep_steps[1])}s {status1}') + + total_reward = ep_rewards.sum() + if total_reward > best_reward: + best_reward = total_reward + model.save(os.path.join(SAVE_DIR, 'best_model')) + log(f' ⭐ NEW BEST: {best_reward:.1f} combined reward') + except Exception as e: + log(f' Eval error: {e}') + import traceback; traceback.print_exc() + +model.save(os.path.join(SAVE_DIR, 'model')) +log(f'\nTraining complete. Best combined reward: {best_reward:.1f}') + +env.close() +time.sleep(5) + +# --- Final eval on all 4 tracks (sequential, port 9091) --- +log('\n' + '=' * 60) +log('FINAL EVALUATION: best_model on 4 tracks (3 sets each)') +log('=' * 60) + +EVAL_TRACKS = [ + ('donkey-generated-track-v0', 'generated_track'), + ('donkey-mountain-track-v0', 'mountain_track'), + ('donkey-minimonaco-track-v0', 'mini_monaco'), + ('donkey-generated-roads-v0', 'generated_road'), +] +EVAL_PORT = 9091 +EVAL_SETS = 3 +EVAL_MAX_STEPS = 2000 + +best_model_path = os.path.join(SAVE_DIR, 'best_model.zip') +results_by_track = {} + +for track_id, track_name in EVAL_TRACKS: + log(f'\n--- {track_name} ---') + steps_list = [] + + for s in range(1, EVAL_SETS + 1): + try: + raw = gym.make(track_id, conf={'host': HOST, 'port': EVAL_PORT}) + inner = ThrottleClampWrapper(raw, throttle_min=THROTTLE_MIN) + inner = StuckTerminationWrapper(inner, stuck_steps=40, min_displacement=0.5) + inner = SpeedRewardWrapper(inner) + eval_env = VecTransposeImage(DummyVecEnv([lambda e=inner: e])) + + eval_model = PPO.load(best_model_path, env=eval_env, device='cpu') + + obs = eval_env.reset() + total_r, total_s, done = 0.0, 0, False + while not done and total_s < EVAL_MAX_STEPS: + action, _ = eval_model.predict(obs, deterministic=True) + result = eval_env.step(action) + if len(result) == 4: + obs, r, d, info = result + done = bool(d[0]) + else: + obs, r, t, tr, info = result + done = bool(t[0] or tr[0]) + total_r += float(r[0]) + total_s += 1 + + status = '✅' if total_s >= EVAL_MAX_STEPS else f'❌@{total_s}' + log(f' Set {s}: {total_r:.1f}r / {total_s}s {status}') + steps_list.append(total_s) + + eval_env.close() + time.sleep(3) + except Exception as e: + log(f' Set {s}: ERROR — {e}') + steps_list.append(0) + time.sleep(3) + + mean_steps = np.mean(steps_list) if steps_list else 0 + results_by_track[track_name] = steps_list + log(f' Mean: {mean_steps:.0f} steps') + +log('\n' + '=' * 60) +log('SUMMARY') +log('=' * 60) +for track_name, steps_list in results_by_track.items(): + steps_str = '/'.join(str(s) for s in steps_list) + mean = np.mean(steps_list) + verdict = '✅' if mean >= 1500 else '⚠️' if mean >= 500 else '❌' + log(f' {verdict} {track_name:20s}: {steps_str} mean={mean:.0f}') + +log(f'\n=== Exp 17 COMPLETE ===') diff --git a/agent/multitrack_runner.py b/agent/multitrack_runner.py index 4aee9df..250d8a5 100644 --- a/agent/multitrack_runner.py +++ b/agent/multitrack_runner.py @@ -143,19 +143,22 @@ class StuckTerminationWrapper(gym.Wrapper): When stuck is detected: terminated=True so SpeedRewardWrapper returns -1.0. """ def __init__(self, env, stuck_steps: int = 80, min_displacement: float = 0.5, - max_stuck_seconds: float = 12.0): + max_stuck_seconds: float = 12.0, max_episode_seconds: float = 30.0): super().__init__(env) - self.stuck_steps = stuck_steps - self.min_displacement = min_displacement - self.max_stuck_seconds = max_stuck_seconds - self._pos_buf: deque = deque(maxlen=stuck_steps) - self._last_progress_pos = None - self._last_progress_t = None + self.stuck_steps = stuck_steps + self.min_displacement = min_displacement + self.max_stuck_seconds = max_stuck_seconds + self.max_episode_seconds = max_episode_seconds + self._pos_buf: deque = deque(maxlen=stuck_steps) + self._last_progress_pos = None + self._last_progress_t = None + self._episode_start_t = None def reset(self, **kwargs): self._pos_buf.clear() self._last_progress_pos = None self._last_progress_t = None + self._episode_start_t = time.time() return self.env.reset(**kwargs) def step(self, action): @@ -194,6 +197,15 @@ class StuckTerminationWrapper(gym.Wrapper): except (TypeError, ValueError): pass + # Hard episode wall-clock limit — fires regardless of car position or sim fps. + # Catches cars sliding slowly along barriers that keep resetting the + # max_stuck_seconds timer by drifting 0.5m at a time. + if not terminated and self._episode_start_t is not None: + if (now - self._episode_start_t) > self.max_episode_seconds: + terminated = True + info['stuck_termination'] = True + info['stuck_reason'] = 'episode_timeout' + # Step-count stuck detection (original logic) if not terminated and len(self._pos_buf) >= self.stuck_steps: displacement = float(np.linalg.norm( diff --git a/monitor_training.sh b/monitor_training.sh new file mode 100644 index 0000000..4ff380b --- /dev/null +++ b/monitor_training.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Standalone training monitor — runs independently of Claude. +# Usage: bash monitor_training.sh +# Output: appended to /tmp/training_monitor.log +# +# Checks every 5 minutes: +# - Is the training process still alive? +# - What are the most recent checkpoint eval scores? +# - Are there any errors or exploit laps? +# - What is the current step count? + +LOG_FILE="${1:-/tmp/exp19.log}" +TRAIN_PID="${2:-}" +MONITOR_OUT="/tmp/training_monitor.log" +INTERVAL=300 # 5 minutes + +echo "======================================" >> "$MONITOR_OUT" +echo "Monitor started: $(date)" >> "$MONITOR_OUT" +echo "Watching: $LOG_FILE PID: $TRAIN_PID" >> "$MONITOR_OUT" +echo "======================================" >> "$MONITOR_OUT" + +while true; do + sleep "$INTERVAL" + echo "" >> "$MONITOR_OUT" + echo "--- $(date) ---" >> "$MONITOR_OUT" + + # Check process alive + if [ -n "$TRAIN_PID" ]; then + if ps -p "$TRAIN_PID" > /dev/null 2>&1; then + echo "Process $TRAIN_PID: RUNNING" >> "$MONITOR_OUT" + else + echo "Process $TRAIN_PID: STOPPED" >> "$MONITOR_OUT" + echo "Training ended at $(date)" >> "$MONITOR_OUT" + break + fi + fi + + # Latest checkpoint/eval/best lines + echo "Recent checkpoints:" >> "$MONITOR_OUT" + grep "Checkpoint\|Eval:\|NEW BEST" "$LOG_FILE" 2>/dev/null | tail -6 >> "$MONITOR_OUT" + + # Step progress + echo "Step progress:" >> "$MONITOR_OUT" + grep "total_timesteps" "$LOG_FILE" 2>/dev/null | tail -1 >> "$MONITOR_OUT" + + # Exploit warning: more than 5 lap times in the last 100 lines + LAP_COUNT=$(tail -100 "$LOG_FILE" 2>/dev/null | grep -c "New lap time") + echo "Laps in last 100 log lines: $LAP_COUNT" >> "$MONITOR_OUT" + if [ "$LAP_COUNT" -gt 10 ]; then + echo "WARNING: high lap count may indicate circular exploit" >> "$MONITOR_OUT" + fi + + # Any errors + ERRORS=$(grep -c "ERROR\|Traceback\|Exception" "$LOG_FILE" 2>/dev/null) + if [ "$ERRORS" -gt 0 ]; then + echo "ERRORS DETECTED: $ERRORS error lines in log" >> "$MONITOR_OUT" + grep "ERROR\|Traceback" "$LOG_FILE" 2>/dev/null | tail -3 >> "$MONITOR_OUT" + fi +done + +echo "Monitor exiting: $(date)" >> "$MONITOR_OUT"