paulh
/
donkeycar-rl-autoresearch
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
donkeycar-rl-autoresearch/agent/experiments/exp22_generated_pair_warm_v...

259 lines
8.5 KiB
Python
Raw Blame History

"""
Exp 22: Parallel DummyVecEnv — generated_road + generated_track, warm-started.
Purpose:
- Keep the generated_road champion warm-start idea.
- Use the full termination stack so wedged cars and circular exploits end fast.
- Use the v6 reward wrapper, which explicitly kills no-progress / low-efficiency
behaviour instead of merely giving it weak reward.
Setup:
- Sim 1: generated_road on port 9091
- Sim 2: generated_track on port 9093
- Warm-start from agent/models/champion/model.zip
"""
import os
import sys
import time
from datetime import datetime
sys.path.insert(0, '/home/paulh/projects/donkeycar-rl-autoresearch/agent')
import gymnasium as gym
import numpy as np
from stable_baselines3 import PPO
from stable_baselines3.common.utils import get_schedule_fn
from stable_baselines3.common.vec_env import DummyVecEnv, VecTransposeImage
from donkeycar_sb3_runner import ThrottleClampWrapper
from multitrack_runner import StuckTerminationWrapper
from reward_wrapper import SpeedRewardWrapper
HOST = 'localhost'
THROTTLE_MIN = 0.2
LR = 0.000225
TOTAL_STEPS = 150_000
CHECKPOINT_EVERY = 10_000
SAVE_DIR = '/home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp22-generated-pair-warm-v6'
WARM_PATH = '/home/paulh/projects/donkeycar-rl-autoresearch/agent/models/champion/model.zip'
os.makedirs(SAVE_DIR, exist_ok=True)
EFFICIENCY_WINDOW = 60
MIN_EFFICIENCY = 0.15
MIN_LAP_TIME = 12.0
MAX_CTE_TERMINATE = 2.5
CTE_PATIENCE = 3
PROGRESS_PATIENCE = 100
EFFICIENCY_PATIENCE = 12
LOW_SPEED_PATIENCE = 10
LOW_SPEED_THRESHOLD = 0.25
LOW_SPEED_MIN_DISPLACEMENT = 0.20
LOW_SPEED_GRACE_STEPS = 15
MAX_STUCK_SECONDS = 3.0
MAX_EPISODE_SECONDS = 18.0
def log(msg):
print(f'[{datetime.now().strftime("%H:%M:%S")}] {msg}', flush=True)
def make_env(track_id, port):
def _init():
raw = gym.make(track_id, conf={'host': HOST, 'port': port})
env = ThrottleClampWrapper(raw, throttle_min=THROTTLE_MIN)
env = StuckTerminationWrapper(
env,
stuck_steps=40,
min_displacement=0.5,
max_stuck_seconds=MAX_STUCK_SECONDS,
max_episode_seconds=MAX_EPISODE_SECONDS,
)
env = SpeedRewardWrapper(
env,
window_size=EFFICIENCY_WINDOW,
min_efficiency=MIN_EFFICIENCY,
min_lap_time=MIN_LAP_TIME,
max_cte_terminate=MAX_CTE_TERMINATE,
cte_patience=CTE_PATIENCE,
progress_patience=PROGRESS_PATIENCE,
efficiency_patience=EFFICIENCY_PATIENCE,
low_speed_patience=LOW_SPEED_PATIENCE,
low_speed_threshold=LOW_SPEED_THRESHOLD,
low_speed_min_displacement=LOW_SPEED_MIN_DISPLACEMENT,
low_speed_grace_steps=LOW_SPEED_GRACE_STEPS,
)
return env
return _init
def make_eval_env(track_id, port):
inner = make_env(track_id, port)()
return VecTransposeImage(DummyVecEnv([lambda e=inner: e]))
log('=' * 60)
log('Exp 22: generated_road + generated_track, warm-started, v6 reward')
log(f' Warm start: {WARM_PATH}')
log(f' Sim 1: {HOST}:9091 -> generated_road')
log(f' Sim 2: {HOST}:9093 -> generated_track')
log(f' throttle_min={THROTTLE_MIN}, lr={LR}, total={TOTAL_STEPS:,}')
log(' Reward: v6 (speed x CTE with progress/efficiency exploit termination)')
log(f' Stuck timeout: {MAX_STUCK_SECONDS:.1f}s, hard cap: {MAX_EPISODE_SECONDS:.1f}s')
log(f' Progress patience: {PROGRESS_PATIENCE} steps')
log(f' Checkpoints: every {CHECKPOINT_EVERY:,} steps')
log('=' * 60)
log('Creating DummyVecEnv with the two road tracks...')
env = DummyVecEnv([
make_env('donkey-generated-roads-v0', 9091),
make_env('donkey-generated-track-v0', 9093),
])
env = VecTransposeImage(env)
log(f' VecEnv num_envs={env.num_envs}, obs={env.observation_space.shape}')
if not os.path.exists(WARM_PATH):
raise FileNotFoundError(WARM_PATH)
model = PPO.load(WARM_PATH, env=env, device='cpu')
model.learning_rate = LR
try:
model.lr_schedule = get_schedule_fn(LR)
except Exception:
model.lr_schedule = None
try:
for pg in model.policy.optimizer.param_groups:
pg['lr'] = LR
except Exception:
pass
log('Warm-start model attached. Starting training...')
best_total_steps = float('-inf')
best_total_reward = float('-inf')
steps_done = 0
while steps_done < TOTAL_STEPS:
seg_steps = min(CHECKPOINT_EVERY, TOTAL_STEPS - steps_done)
model.learn(total_timesteps=seg_steps, reset_num_timesteps=False)
steps_done += seg_steps
ckpt = os.path.join(SAVE_DIR, f'checkpoint_{steps_done:07d}')
model.save(ckpt)
model.save(os.path.join(SAVE_DIR, 'model'))
log(f'[{steps_done:,}/{TOTAL_STEPS:,}] Checkpoint saved: {ckpt}.zip')
try:
obs = env.reset()
ep_rewards = np.zeros(env.num_envs)
ep_steps = np.zeros(env.num_envs)
done_mask = np.zeros(env.num_envs, dtype=bool)
for _ in range(2000):
action, _ = model.predict(obs, deterministic=True)
obs, rewards, dones, infos = env.step(action)
for i in range(env.num_envs):
if not done_mask[i]:
ep_rewards[i] += rewards[i]
ep_steps[i] += 1
if dones[i]:
done_mask[i] = True
if done_mask.all():
break
status0 = '✅' if ep_steps[0] >= 2000 else f'❌@{int(ep_steps[0])}'
status1 = '✅' if ep_steps[1] >= 2000 else f'❌@{int(ep_steps[1])}'
log(
f' Eval: gen_road={ep_rewards[0]:.1f}r/{int(ep_steps[0])}s {status0} '
f'gen_track={ep_rewards[1]:.1f}r/{int(ep_steps[1])}s {status1}'
)
total_steps_eval = ep_steps.sum()
total_reward = ep_rewards.sum()
if (
total_steps_eval > best_total_steps
or (total_steps_eval == best_total_steps and total_reward > best_total_reward)
):
best_total_steps = total_steps_eval
best_total_reward = total_reward
model.save(os.path.join(SAVE_DIR, 'best_model'))
log(
f' NEW BEST: combined steps={int(best_total_steps)} '
f'reward={best_total_reward:.1f}'
)
except Exception as e:
log(f' Eval error: {e}')
import traceback
traceback.print_exc()
model.save(os.path.join(SAVE_DIR, 'model'))
log(f'\nTraining complete. Best combined steps: {int(best_total_steps)}')
env.close()
time.sleep(5)
log('\n' + '=' * 60)
log('FINAL EVALUATION: best_model on generated_road, generated_track, mini_monaco')
log('=' * 60)
EVAL_TRACKS = [
('donkey-generated-roads-v0', 'generated_road'),
('donkey-generated-track-v0', 'generated_track'),
('donkey-minimonaco-track-v0', 'mini_monaco'),
]
EVAL_PORT = 9091
EVAL_SETS = 3
EVAL_MAX_STEPS = 2000
best_model_path = os.path.join(SAVE_DIR, 'best_model.zip')
results_by_track = {}
for track_id, track_name in EVAL_TRACKS:
log(f'\n--- {track_name} ---')
steps_list = []
for s in range(1, EVAL_SETS + 1):
try:
eval_env = make_eval_env(track_id, EVAL_PORT)
eval_model = PPO.load(best_model_path, env=eval_env, device='cpu')
obs = eval_env.reset()
total_r, total_s, done = 0.0, 0, False
while not done and total_s < EVAL_MAX_STEPS:
action, _ = eval_model.predict(obs, deterministic=True)
result = eval_env.step(action)
if len(result) == 4:
obs, r, d, info = result
done = bool(d[0])
else:
obs, r, t, tr, info = result
done = bool(t[0] or tr[0])
total_r += float(r[0])
total_s += 1
status = '✅' if total_s >= EVAL_MAX_STEPS else f'❌@{total_s}'
log(f' Set {s}: {total_r:.1f}r / {total_s}s {status}')
steps_list.append(total_s)
eval_env.close()
time.sleep(3)
except Exception as e:
log(f' Set {s}: ERROR — {e}')
steps_list.append(0)
time.sleep(3)
mean_steps = np.mean(steps_list) if steps_list else 0
results_by_track[track_name] = steps_list
log(f' Mean: {mean_steps:.0f} steps')
log('\n' + '=' * 60)
log('SUMMARY')
log('=' * 60)
for track_name, steps_list in results_by_track.items():
steps_str = '/'.join(str(s) for s in steps_list)
mean = np.mean(steps_list)
verdict = '✅' if mean >= 1500 else '⚠️' if mean >= 500 else '❌'
log(f' {verdict} {track_name:20s}: {steps_str} mean={mean:.0f}')
log('\n=== Exp 22 COMPLETE ===')
Reference in New Issue View Git Blame Copy Permalink