fix: exp19 — hard episode time limit to stop minutes-long stuck cars
StuckTerminationWrapper wall-clock timer was resettable by barrier-sliding: car drifting 0.5m along a wall repeatedly resets the 12s timer. At low sim fps (1-2fps when both cars stuck), 40-step check also takes minutes. Fix: added max_episode_seconds=30 — hard wall-clock limit per episode, independent of position or sim fps. No episode can run longer than 30s. Also adds monitor_training.sh: independent shell process that checks every 5 minutes and appends status to /tmp/training_monitor.log — works without Claude being active. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Paul Huliganga
parent
7fdfbacaee
commit
04d5a10992
|
|
@ -0,0 +1,200 @@
|
|||
"""
|
||||
Exp 19: Parallel DummyVecEnv — 450k steps, fixed stuck detection (v3).
|
||||
|
||||
Fixes from Exp 18:
|
||||
- Exp 17 fix carried forward: window_size=200, min_lap_time=12s
|
||||
|
||||
New fix for Exp 19:
|
||||
StuckTerminationWrapper wall-clock timer was resettable: a car slowly
|
||||
sliding along a barrier can drift 0.5m repeatedly, resetting the 12s
|
||||
timer indefinitely. At low sim fps (1-2fps when both cars stuck against
|
||||
walls), even the 40-step check takes minutes. Cars were stuck visually
|
||||
for minutes with no episode termination.
|
||||
|
||||
Fix: hard max_episode_seconds=30 in StuckTerminationWrapper. Every
|
||||
episode terminates after 30 wall-clock seconds regardless of car
|
||||
position, sim fps, or barrier sliding. No episode can stall longer.
|
||||
|
||||
Setup — TWO sim instances required:
|
||||
Sim 1: donkey_sim.exe on port 9091 → generated_track
|
||||
Sim 2: separate copy of donkey_sim.exe on port 9093 → mountain_track
|
||||
"""
|
||||
import sys, os, time
|
||||
sys.path.insert(0, '/home/paulh/projects/donkeycar-rl-autoresearch/agent')
|
||||
|
||||
from multitrack_runner import log, StuckTerminationWrapper
|
||||
from donkeycar_sb3_runner import ThrottleClampWrapper
|
||||
from reward_wrapper import SpeedRewardWrapper
|
||||
from stable_baselines3 import PPO
|
||||
from stable_baselines3.common.vec_env import DummyVecEnv, VecTransposeImage
|
||||
import gymnasium as gym
|
||||
import numpy as np
|
||||
|
||||
HOST = 'localhost'
|
||||
THROTTLE_MIN = 0.2
|
||||
LR = 0.000725
|
||||
TOTAL_STEPS = 450_000
|
||||
CHECKPOINT_EVERY = 20_000
|
||||
SAVE_DIR = '/home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp19-parallel-450k-v3'
|
||||
os.makedirs(SAVE_DIR, exist_ok=True)
|
||||
|
||||
# Exploit fixes: larger window catches full circles; higher min_lap_time
|
||||
# kills sub-genuine laps before they can contribute positive reward.
|
||||
EFFICIENCY_WINDOW = 200 # was 30 — now covers 2+ exploit circles at 22fps
|
||||
MIN_LAP_TIME = 12.0 # was 5.0 — genuine laps: gentrack 13-16s, mountain 27-29s
|
||||
|
||||
|
||||
def make_env(track_id, port):
|
||||
def _init():
|
||||
raw = gym.make(track_id, conf={'host': HOST, 'port': port})
|
||||
env = ThrottleClampWrapper(raw, throttle_min=THROTTLE_MIN)
|
||||
env = StuckTerminationWrapper(env, stuck_steps=40, min_displacement=0.5,
|
||||
max_episode_seconds=30.0)
|
||||
env = SpeedRewardWrapper(env, window_size=EFFICIENCY_WINDOW, min_lap_time=MIN_LAP_TIME)
|
||||
return env
|
||||
return _init
|
||||
|
||||
|
||||
log('=' * 60)
|
||||
log('Exp 19: Parallel DummyVecEnv — 450k steps (stuck fix v3)')
|
||||
log(f' Sim 1: {HOST}:9091 → generated_track')
|
||||
log(f' Sim 2: {HOST}:9093 → mountain_track')
|
||||
log(f' throttle_min={THROTTLE_MIN}, lr={LR}, total={TOTAL_STEPS:,}')
|
||||
log(f' Reward: v6 + exploit fix (window={EFFICIENCY_WINDOW}, min_lap={MIN_LAP_TIME}s)')
|
||||
log(f' Stuck termination: 40 steps (~2.5s)')
|
||||
log(f' Checkpoints: every {CHECKPOINT_EVERY:,} steps')
|
||||
log('=' * 60)
|
||||
|
||||
log('Creating DummyVecEnv with two tracks...')
|
||||
env = DummyVecEnv([
|
||||
make_env('donkey-generated-track-v0', 9091),
|
||||
make_env('donkey-mountain-track-v0', 9093),
|
||||
])
|
||||
env = VecTransposeImage(env)
|
||||
log(f' VecEnv num_envs={env.num_envs}, obs={env.observation_space.shape}')
|
||||
|
||||
model = PPO('CnnPolicy', env, learning_rate=LR, verbose=1, device='cpu')
|
||||
log('PPO created. Starting training...')
|
||||
|
||||
best_reward = float('-inf')
|
||||
steps_done = 0
|
||||
|
||||
while steps_done < TOTAL_STEPS:
|
||||
seg_steps = min(CHECKPOINT_EVERY, TOTAL_STEPS - steps_done)
|
||||
model.learn(total_timesteps=seg_steps, reset_num_timesteps=False)
|
||||
steps_done += seg_steps
|
||||
|
||||
ckpt = os.path.join(SAVE_DIR, f'checkpoint_{steps_done:07d}')
|
||||
model.save(ckpt)
|
||||
model.save(os.path.join(SAVE_DIR, 'model'))
|
||||
log(f'[{steps_done:,}/{TOTAL_STEPS:,}] Checkpoint saved: {ckpt}.zip')
|
||||
|
||||
# Eval on both training tracks using the existing DummyVecEnv connections
|
||||
try:
|
||||
obs = env.reset()
|
||||
ep_rewards = np.zeros(env.num_envs)
|
||||
ep_steps = np.zeros(env.num_envs)
|
||||
done_mask = np.zeros(env.num_envs, dtype=bool)
|
||||
for _ in range(2000):
|
||||
action, _ = model.predict(obs, deterministic=True)
|
||||
obs, rewards, dones, infos = env.step(action)
|
||||
for i in range(env.num_envs):
|
||||
if not done_mask[i]:
|
||||
ep_rewards[i] += rewards[i]
|
||||
ep_steps[i] += 1
|
||||
if dones[i]:
|
||||
done_mask[i] = True
|
||||
if done_mask.all():
|
||||
break
|
||||
|
||||
status0 = '✅' if ep_steps[0] >= 2000 else f'❌@{int(ep_steps[0])}'
|
||||
status1 = '✅' if ep_steps[1] >= 2000 else f'❌@{int(ep_steps[1])}'
|
||||
log(f' Eval: gen_track={ep_rewards[0]:.1f}r/{int(ep_steps[0])}s {status0} '
|
||||
f'mountain={ep_rewards[1]:.1f}r/{int(ep_steps[1])}s {status1}')
|
||||
|
||||
total_reward = ep_rewards.sum()
|
||||
if total_reward > best_reward:
|
||||
best_reward = total_reward
|
||||
model.save(os.path.join(SAVE_DIR, 'best_model'))
|
||||
log(f' ⭐ NEW BEST: {best_reward:.1f} combined reward')
|
||||
except Exception as e:
|
||||
log(f' Eval error: {e}')
|
||||
import traceback; traceback.print_exc()
|
||||
|
||||
model.save(os.path.join(SAVE_DIR, 'model'))
|
||||
log(f'\nTraining complete. Best combined reward: {best_reward:.1f}')
|
||||
|
||||
env.close()
|
||||
time.sleep(5)
|
||||
|
||||
# --- Final eval on all 4 tracks (sequential, port 9091) ---
|
||||
log('\n' + '=' * 60)
|
||||
log('FINAL EVALUATION: best_model on 4 tracks (3 sets each)')
|
||||
log('=' * 60)
|
||||
|
||||
EVAL_TRACKS = [
|
||||
('donkey-generated-track-v0', 'generated_track'),
|
||||
('donkey-mountain-track-v0', 'mountain_track'),
|
||||
('donkey-minimonaco-track-v0', 'mini_monaco'),
|
||||
('donkey-generated-roads-v0', 'generated_road'),
|
||||
]
|
||||
EVAL_PORT = 9091
|
||||
EVAL_SETS = 3
|
||||
EVAL_MAX_STEPS = 2000
|
||||
|
||||
best_model_path = os.path.join(SAVE_DIR, 'best_model.zip')
|
||||
results_by_track = {}
|
||||
|
||||
for track_id, track_name in EVAL_TRACKS:
|
||||
log(f'\n--- {track_name} ---')
|
||||
steps_list = []
|
||||
|
||||
for s in range(1, EVAL_SETS + 1):
|
||||
try:
|
||||
raw = gym.make(track_id, conf={'host': HOST, 'port': EVAL_PORT})
|
||||
inner = ThrottleClampWrapper(raw, throttle_min=THROTTLE_MIN)
|
||||
inner = StuckTerminationWrapper(inner, stuck_steps=40, min_displacement=0.5)
|
||||
inner = SpeedRewardWrapper(inner)
|
||||
eval_env = VecTransposeImage(DummyVecEnv([lambda e=inner: e]))
|
||||
|
||||
eval_model = PPO.load(best_model_path, env=eval_env, device='cpu')
|
||||
|
||||
obs = eval_env.reset()
|
||||
total_r, total_s, done = 0.0, 0, False
|
||||
while not done and total_s < EVAL_MAX_STEPS:
|
||||
action, _ = eval_model.predict(obs, deterministic=True)
|
||||
result = eval_env.step(action)
|
||||
if len(result) == 4:
|
||||
obs, r, d, info = result
|
||||
done = bool(d[0])
|
||||
else:
|
||||
obs, r, t, tr, info = result
|
||||
done = bool(t[0] or tr[0])
|
||||
total_r += float(r[0])
|
||||
total_s += 1
|
||||
|
||||
status = '✅' if total_s >= EVAL_MAX_STEPS else f'❌@{total_s}'
|
||||
log(f' Set {s}: {total_r:.1f}r / {total_s}s {status}')
|
||||
steps_list.append(total_s)
|
||||
|
||||
eval_env.close()
|
||||
time.sleep(3)
|
||||
except Exception as e:
|
||||
log(f' Set {s}: ERROR — {e}')
|
||||
steps_list.append(0)
|
||||
time.sleep(3)
|
||||
|
||||
mean_steps = np.mean(steps_list) if steps_list else 0
|
||||
results_by_track[track_name] = steps_list
|
||||
log(f' Mean: {mean_steps:.0f} steps')
|
||||
|
||||
log('\n' + '=' * 60)
|
||||
log('SUMMARY')
|
||||
log('=' * 60)
|
||||
for track_name, steps_list in results_by_track.items():
|
||||
steps_str = '/'.join(str(s) for s in steps_list)
|
||||
mean = np.mean(steps_list)
|
||||
verdict = '✅' if mean >= 1500 else '⚠️' if mean >= 500 else '❌'
|
||||
log(f' {verdict} {track_name:20s}: {steps_str} mean={mean:.0f}')
|
||||
|
||||
log(f'\n=== Exp 17 COMPLETE ===')
|
||||
|
|
@ -143,19 +143,22 @@ class StuckTerminationWrapper(gym.Wrapper):
|
|||
When stuck is detected: terminated=True so SpeedRewardWrapper returns -1.0.
|
||||
"""
|
||||
def __init__(self, env, stuck_steps: int = 80, min_displacement: float = 0.5,
|
||||
max_stuck_seconds: float = 12.0):
|
||||
max_stuck_seconds: float = 12.0, max_episode_seconds: float = 30.0):
|
||||
super().__init__(env)
|
||||
self.stuck_steps = stuck_steps
|
||||
self.min_displacement = min_displacement
|
||||
self.max_stuck_seconds = max_stuck_seconds
|
||||
self._pos_buf: deque = deque(maxlen=stuck_steps)
|
||||
self._last_progress_pos = None
|
||||
self._last_progress_t = None
|
||||
self.stuck_steps = stuck_steps
|
||||
self.min_displacement = min_displacement
|
||||
self.max_stuck_seconds = max_stuck_seconds
|
||||
self.max_episode_seconds = max_episode_seconds
|
||||
self._pos_buf: deque = deque(maxlen=stuck_steps)
|
||||
self._last_progress_pos = None
|
||||
self._last_progress_t = None
|
||||
self._episode_start_t = None
|
||||
|
||||
def reset(self, **kwargs):
|
||||
self._pos_buf.clear()
|
||||
self._last_progress_pos = None
|
||||
self._last_progress_t = None
|
||||
self._episode_start_t = time.time()
|
||||
return self.env.reset(**kwargs)
|
||||
|
||||
def step(self, action):
|
||||
|
|
@ -194,6 +197,15 @@ class StuckTerminationWrapper(gym.Wrapper):
|
|||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
# Hard episode wall-clock limit — fires regardless of car position or sim fps.
|
||||
# Catches cars sliding slowly along barriers that keep resetting the
|
||||
# max_stuck_seconds timer by drifting 0.5m at a time.
|
||||
if not terminated and self._episode_start_t is not None:
|
||||
if (now - self._episode_start_t) > self.max_episode_seconds:
|
||||
terminated = True
|
||||
info['stuck_termination'] = True
|
||||
info['stuck_reason'] = 'episode_timeout'
|
||||
|
||||
# Step-count stuck detection (original logic)
|
||||
if not terminated and len(self._pos_buf) >= self.stuck_steps:
|
||||
displacement = float(np.linalg.norm(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,61 @@
|
|||
#!/bin/bash
|
||||
# Standalone training monitor — runs independently of Claude.
|
||||
# Usage: bash monitor_training.sh <log_file> <pid>
|
||||
# Output: appended to /tmp/training_monitor.log
|
||||
#
|
||||
# Checks every 5 minutes:
|
||||
# - Is the training process still alive?
|
||||
# - What are the most recent checkpoint eval scores?
|
||||
# - Are there any errors or exploit laps?
|
||||
# - What is the current step count?
|
||||
|
||||
LOG_FILE="${1:-/tmp/exp19.log}"
|
||||
TRAIN_PID="${2:-}"
|
||||
MONITOR_OUT="/tmp/training_monitor.log"
|
||||
INTERVAL=300 # 5 minutes
|
||||
|
||||
echo "======================================" >> "$MONITOR_OUT"
|
||||
echo "Monitor started: $(date)" >> "$MONITOR_OUT"
|
||||
echo "Watching: $LOG_FILE PID: $TRAIN_PID" >> "$MONITOR_OUT"
|
||||
echo "======================================" >> "$MONITOR_OUT"
|
||||
|
||||
while true; do
|
||||
sleep "$INTERVAL"
|
||||
echo "" >> "$MONITOR_OUT"
|
||||
echo "--- $(date) ---" >> "$MONITOR_OUT"
|
||||
|
||||
# Check process alive
|
||||
if [ -n "$TRAIN_PID" ]; then
|
||||
if ps -p "$TRAIN_PID" > /dev/null 2>&1; then
|
||||
echo "Process $TRAIN_PID: RUNNING" >> "$MONITOR_OUT"
|
||||
else
|
||||
echo "Process $TRAIN_PID: STOPPED" >> "$MONITOR_OUT"
|
||||
echo "Training ended at $(date)" >> "$MONITOR_OUT"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
|
||||
# Latest checkpoint/eval/best lines
|
||||
echo "Recent checkpoints:" >> "$MONITOR_OUT"
|
||||
grep "Checkpoint\|Eval:\|NEW BEST" "$LOG_FILE" 2>/dev/null | tail -6 >> "$MONITOR_OUT"
|
||||
|
||||
# Step progress
|
||||
echo "Step progress:" >> "$MONITOR_OUT"
|
||||
grep "total_timesteps" "$LOG_FILE" 2>/dev/null | tail -1 >> "$MONITOR_OUT"
|
||||
|
||||
# Exploit warning: more than 5 lap times in the last 100 lines
|
||||
LAP_COUNT=$(tail -100 "$LOG_FILE" 2>/dev/null | grep -c "New lap time")
|
||||
echo "Laps in last 100 log lines: $LAP_COUNT" >> "$MONITOR_OUT"
|
||||
if [ "$LAP_COUNT" -gt 10 ]; then
|
||||
echo "WARNING: high lap count may indicate circular exploit" >> "$MONITOR_OUT"
|
||||
fi
|
||||
|
||||
# Any errors
|
||||
ERRORS=$(grep -c "ERROR\|Traceback\|Exception" "$LOG_FILE" 2>/dev/null)
|
||||
if [ "$ERRORS" -gt 0 ]; then
|
||||
echo "ERRORS DETECTED: $ERRORS error lines in log" >> "$MONITOR_OUT"
|
||||
grep "ERROR\|Traceback" "$LOG_FILE" 2>/dev/null | tail -3 >> "$MONITOR_OUT"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Monitor exiting: $(date)" >> "$MONITOR_OUT"
|
||||
Loading…
Reference in New Issue