292 lines
9.9 KiB
Python
292 lines
9.9 KiB
Python
"""
|
|
Exp 21: Parallel DummyVecEnv — generated_road + generated_track, warm-started.
|
|
|
|
Rationale:
|
|
- generated_road specialist already exists and drives road markings well.
|
|
- generated_road and generated_track share the same road semantics.
|
|
- Background adaptation is the goal here, not mountain physics.
|
|
|
|
Design:
|
|
- Warm-start from Phase 2 champion (generated_road specialist).
|
|
- Train in parallel on TWO sim instances:
|
|
Sim 1: generated_road on port 9091
|
|
Sim 2: generated_track on port 9093
|
|
- Use the old v4 reward that worked for the flat road tracks.
|
|
- Keep the wrapper chain minimal: ThrottleClamp + V4 reward only.
|
|
"""
|
|
import sys, os, time
|
|
from collections import deque
|
|
from datetime import datetime
|
|
|
|
sys.path.insert(0, '/home/paulh/projects/donkeycar-rl-autoresearch/agent')
|
|
|
|
from donkeycar_sb3_runner import ThrottleClampWrapper
|
|
from multitrack_runner import StuckTerminationWrapper
|
|
from stable_baselines3 import PPO
|
|
from stable_baselines3.common.vec_env import DummyVecEnv, VecTransposeImage
|
|
from stable_baselines3.common.utils import get_schedule_fn
|
|
import gymnasium as gym
|
|
import numpy as np
|
|
|
|
|
|
HOST = 'localhost'
|
|
THROTTLE_MIN = 0.2
|
|
LR = 0.000225
|
|
TOTAL_STEPS = 150_000
|
|
CHECKPOINT_EVERY = 10_000
|
|
SAVE_DIR = '/home/paulh/projects/donkeycar-rl-autoresearch/agent/models/exp21-generated-pair-warm-v4'
|
|
WARM_PATH = '/home/paulh/projects/donkeycar-rl-autoresearch/agent/models/champion/model.zip'
|
|
os.makedirs(SAVE_DIR, exist_ok=True)
|
|
|
|
|
|
class V4RewardWrapper(gym.Wrapper):
|
|
"""
|
|
v4 reward from the successful flat-road experiments:
|
|
reward = base_cte * efficiency * (1 + speed_scale * speed)
|
|
"""
|
|
def __init__(self, env, speed_scale=0.1, window_size=60,
|
|
min_efficiency=0.05, max_cte=8.0):
|
|
super().__init__(env)
|
|
self.speed_scale = speed_scale
|
|
self.min_efficiency = min_efficiency
|
|
self.max_cte = max_cte
|
|
self._pos_history = deque(maxlen=window_size + 1)
|
|
|
|
def reset(self, **kwargs):
|
|
self._pos_history.clear()
|
|
return self.env.reset(**kwargs)
|
|
|
|
def step(self, action):
|
|
result = self.env.step(action)
|
|
if len(result) == 5:
|
|
obs, _sim_reward, terminated, truncated, info = result
|
|
done = terminated or truncated
|
|
else:
|
|
obs, _sim_reward, done, info = result
|
|
terminated, truncated = done, False
|
|
|
|
reward = self._compute_reward(done, info)
|
|
|
|
if len(result) == 5:
|
|
return obs, reward, terminated, truncated, info
|
|
return obs, reward, done, info
|
|
|
|
def _compute_reward(self, done, info):
|
|
if done:
|
|
return -1.0
|
|
|
|
pos = info.get('pos', None)
|
|
if pos is not None:
|
|
try:
|
|
self._pos_history.append(np.array(list(pos)[:3], dtype=np.float64))
|
|
except (TypeError, ValueError):
|
|
pass
|
|
|
|
try:
|
|
cte = float(info.get('cte', 0.0) or 0.0)
|
|
except (TypeError, ValueError):
|
|
cte = 0.0
|
|
base = 1.0 - min(abs(cte) / self.max_cte, 1.0)
|
|
|
|
efficiency = self._compute_efficiency()
|
|
eff = max(0.0, (efficiency - self.min_efficiency) / (1.0 - self.min_efficiency))
|
|
|
|
try:
|
|
speed = max(0.0, float(info.get('speed', 0.0) or 0.0))
|
|
except (TypeError, ValueError):
|
|
speed = 0.0
|
|
|
|
return base * eff * (1.0 + self.speed_scale * speed)
|
|
|
|
def _compute_efficiency(self):
|
|
if len(self._pos_history) < 3:
|
|
return 1.0
|
|
positions = list(self._pos_history)
|
|
net = np.linalg.norm(positions[-1] - positions[0])
|
|
total = sum(
|
|
np.linalg.norm(positions[i + 1] - positions[i])
|
|
for i in range(len(positions) - 1)
|
|
)
|
|
return float(net / total) if total > 1e-6 else 1.0
|
|
|
|
|
|
def log(msg):
|
|
print(f'[{datetime.now().strftime("%H:%M:%S")}] {msg}', flush=True)
|
|
|
|
|
|
def make_env(track_id, port):
|
|
def _init():
|
|
raw = gym.make(track_id, conf={'host': HOST, 'port': port})
|
|
env = ThrottleClampWrapper(raw, throttle_min=THROTTLE_MIN)
|
|
env = StuckTerminationWrapper(
|
|
env,
|
|
stuck_steps=40,
|
|
min_displacement=0.5,
|
|
max_stuck_seconds=12.0,
|
|
max_episode_seconds=30.0,
|
|
)
|
|
env = V4RewardWrapper(env, speed_scale=0.1, window_size=60,
|
|
min_efficiency=0.05, max_cte=8.0)
|
|
return env
|
|
return _init
|
|
|
|
|
|
def make_eval_env(track_id, port):
|
|
inner = make_env(track_id, port)()
|
|
return VecTransposeImage(DummyVecEnv([lambda e=inner: e]))
|
|
|
|
|
|
log('=' * 60)
|
|
log('Exp 21: generated_road + generated_track, warm-started, v4 reward')
|
|
log(f' Warm start: {WARM_PATH}')
|
|
log(f' Sim 1: {HOST}:9091 -> generated_road')
|
|
log(f' Sim 2: {HOST}:9093 -> generated_track')
|
|
log(f' throttle_min={THROTTLE_MIN}, lr={LR}, total={TOTAL_STEPS:,}')
|
|
log(' Termination: StuckTerminationWrapper enabled')
|
|
log(f' Checkpoints: every {CHECKPOINT_EVERY:,} steps')
|
|
log('=' * 60)
|
|
|
|
log('Creating DummyVecEnv with the two road tracks...')
|
|
env = DummyVecEnv([
|
|
make_env('donkey-generated-roads-v0', 9091),
|
|
make_env('donkey-generated-track-v0', 9093),
|
|
])
|
|
env = VecTransposeImage(env)
|
|
log(f' VecEnv num_envs={env.num_envs}, obs={env.observation_space.shape}')
|
|
|
|
if not os.path.exists(WARM_PATH):
|
|
raise FileNotFoundError(WARM_PATH)
|
|
|
|
model = PPO.load(WARM_PATH, env=env, device='cpu')
|
|
model.learning_rate = LR
|
|
try:
|
|
model.lr_schedule = get_schedule_fn(LR)
|
|
except Exception:
|
|
model.lr_schedule = None
|
|
try:
|
|
for pg in model.policy.optimizer.param_groups:
|
|
pg['lr'] = LR
|
|
except Exception:
|
|
pass
|
|
|
|
log('Warm-start model attached. Starting training...')
|
|
|
|
best_total_steps = float('-inf')
|
|
best_total_reward = float('-inf')
|
|
steps_done = 0
|
|
|
|
while steps_done < TOTAL_STEPS:
|
|
seg_steps = min(CHECKPOINT_EVERY, TOTAL_STEPS - steps_done)
|
|
model.learn(total_timesteps=seg_steps, reset_num_timesteps=False)
|
|
steps_done += seg_steps
|
|
|
|
ckpt = os.path.join(SAVE_DIR, f'checkpoint_{steps_done:07d}')
|
|
model.save(ckpt)
|
|
model.save(os.path.join(SAVE_DIR, 'model'))
|
|
log(f'[{steps_done:,}/{TOTAL_STEPS:,}] Checkpoint saved: {ckpt}.zip')
|
|
|
|
try:
|
|
obs = env.reset()
|
|
ep_rewards = np.zeros(env.num_envs)
|
|
ep_steps = np.zeros(env.num_envs)
|
|
done_mask = np.zeros(env.num_envs, dtype=bool)
|
|
|
|
for _ in range(2000):
|
|
action, _ = model.predict(obs, deterministic=True)
|
|
obs, rewards, dones, infos = env.step(action)
|
|
for i in range(env.num_envs):
|
|
if not done_mask[i]:
|
|
ep_rewards[i] += rewards[i]
|
|
ep_steps[i] += 1
|
|
if dones[i]:
|
|
done_mask[i] = True
|
|
if done_mask.all():
|
|
break
|
|
|
|
status0 = '✅' if ep_steps[0] >= 2000 else f'❌@{int(ep_steps[0])}'
|
|
status1 = '✅' if ep_steps[1] >= 2000 else f'❌@{int(ep_steps[1])}'
|
|
log(f' Eval: gen_road={ep_rewards[0]:.1f}r/{int(ep_steps[0])}s {status0} '
|
|
f'gen_track={ep_rewards[1]:.1f}r/{int(ep_steps[1])}s {status1}')
|
|
|
|
total_steps_eval = ep_steps.sum()
|
|
total_reward = ep_rewards.sum()
|
|
if (total_steps_eval > best_total_steps or
|
|
(total_steps_eval == best_total_steps and total_reward > best_total_reward)):
|
|
best_total_steps = total_steps_eval
|
|
best_total_reward = total_reward
|
|
model.save(os.path.join(SAVE_DIR, 'best_model'))
|
|
log(f' NEW BEST: combined steps={int(best_total_steps)} reward={best_total_reward:.1f}')
|
|
except Exception as e:
|
|
log(f' Eval error: {e}')
|
|
import traceback; traceback.print_exc()
|
|
|
|
model.save(os.path.join(SAVE_DIR, 'model'))
|
|
log(f'\nTraining complete. Best combined steps: {int(best_total_steps)}')
|
|
|
|
env.close()
|
|
time.sleep(5)
|
|
|
|
log('\n' + '=' * 60)
|
|
log('FINAL EVALUATION: best_model on generated_road, generated_track, mini_monaco')
|
|
log('=' * 60)
|
|
|
|
EVAL_TRACKS = [
|
|
('donkey-generated-roads-v0', 'generated_road'),
|
|
('donkey-generated-track-v0', 'generated_track'),
|
|
('donkey-minimonaco-track-v0', 'mini_monaco'),
|
|
]
|
|
EVAL_PORT = 9091
|
|
EVAL_SETS = 3
|
|
EVAL_MAX_STEPS = 2000
|
|
|
|
best_model_path = os.path.join(SAVE_DIR, 'best_model.zip')
|
|
results_by_track = {}
|
|
|
|
for track_id, track_name in EVAL_TRACKS:
|
|
log(f'\n--- {track_name} ---')
|
|
steps_list = []
|
|
|
|
for s in range(1, EVAL_SETS + 1):
|
|
try:
|
|
eval_env = make_eval_env(track_id, EVAL_PORT)
|
|
eval_model = PPO.load(best_model_path, env=eval_env, device='cpu')
|
|
|
|
obs = eval_env.reset()
|
|
total_r, total_s, done = 0.0, 0, False
|
|
while not done and total_s < EVAL_MAX_STEPS:
|
|
action, _ = eval_model.predict(obs, deterministic=True)
|
|
result = eval_env.step(action)
|
|
if len(result) == 4:
|
|
obs, r, d, info = result
|
|
done = bool(d[0])
|
|
else:
|
|
obs, r, t, tr, info = result
|
|
done = bool(t[0] or tr[0])
|
|
total_r += float(r[0])
|
|
total_s += 1
|
|
|
|
status = '✅' if total_s >= EVAL_MAX_STEPS else f'❌@{total_s}'
|
|
log(f' Set {s}: {total_r:.1f}r / {total_s}s {status}')
|
|
steps_list.append(total_s)
|
|
eval_env.close()
|
|
time.sleep(3)
|
|
except Exception as e:
|
|
log(f' Set {s}: ERROR - {e}')
|
|
steps_list.append(0)
|
|
time.sleep(3)
|
|
|
|
mean_steps = np.mean(steps_list) if steps_list else 0
|
|
results_by_track[track_name] = steps_list
|
|
log(f' Mean: {mean_steps:.0f} steps')
|
|
|
|
log('\n' + '=' * 60)
|
|
log('SUMMARY')
|
|
log('=' * 60)
|
|
for track_name, steps_list in results_by_track.items():
|
|
steps_str = '/'.join(str(s) for s in steps_list)
|
|
mean = np.mean(steps_list)
|
|
verdict = '✅' if mean >= 1500 else '⚠️' if mean >= 500 else '❌'
|
|
log(f' {verdict} {track_name:20s}: {steps_str} mean={mean:.0f}')
|
|
|
|
log('\n=== Exp 21 COMPLETE ===')
|