#!/usr/bin/env ts-node /** * scripts/model-report.ts * * Parses git log trailers to generate a model performance table. * Reads: Agent, Tests, Tests-Added, TypeScript trailers from commit messages. * * Usage: * npx ts-node scripts/model-report.ts * npx ts-node scripts/model-report.ts --since=2026-03-01 * npx ts-node scripts/model-report.ts --json */ import { execSync } from 'child_process'; interface CommitRecord { hash: string; date: string; subject: string; agent: string; tests: string; // raw e.g. "129/129 passing" testsAdded: string; // raw e.g. "+32" typescript: string; // raw e.g. "clean" or "2 errors" body: string; } interface ModelStats { model: string; commits: number; testsAdded: number; typeScriptErrors: number; // errors introduced typeScriptClean: number; // commits that were clean hasAttribution: number; // commits with Agent: trailer commitsByType: Record; firstSeen: string; lastSeen: string; subjects: string[]; } function parseArgs() { const args = process.argv.slice(2); const since = args.find(a => a.startsWith('--since='))?.split('=')[1]; const asJson = args.includes('--json'); return { since, asJson }; } function getCommits(since?: string): CommitRecord[] { const sinceFlag = since ? `--since="${since}"` : ''; // Use %x00 as field separator, %x01 as record separator const format = '%H%x00%ad%x00%s%x00%b%x01'; const cmd = `git log ${sinceFlag} --format="${format}" --date=short`; let raw: string; try { raw = execSync(cmd, { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 }); } catch { console.error('Failed to run git log'); process.exit(1); } const records = raw.split('\x01').filter(r => r.trim()); return records.map(record => { const [hash, date, subject, body = ''] = record.split('\x00'); const trailer = (key: string) => { const match = body.match(new RegExp(`^${key}:\\s*(.+)$`, 'm')); return match ? match[1].trim() : ''; }; return { hash: (hash || '').trim().slice(0, 7), date: (date || '').trim(), subject: (subject || '').trim(), agent: trailer('Agent'), tests: trailer('Tests'), testsAdded: trailer('Tests-Added'), typescript: trailer('TypeScript'), body: body.trim(), }; }).filter(r => r.hash); } function parseTestsAdded(raw: string): number { if (!raw) return -1; // -1 = unknown (no trailer) const match = raw.match(/[+-]?(\d+)/); return match ? parseInt(match[1], 10) : 0; } function parseTsErrors(raw: string): number { if (!raw) return -1; // -1 = unknown if (raw.toLowerCase().includes('clean')) return 0; const match = raw.match(/(\d+)/); return match ? parseInt(match[1], 10) : -1; } function commitType(subject: string): string { const match = subject.match(/^(feat|fix|refactor|test|docs|chore|build|ci|perf)/); return match ? match[1] : 'other'; } function buildStats(commits: CommitRecord[]): Map { const stats = new Map(); for (const c of commits) { const model = c.agent || 'unknown'; if (!stats.has(model)) { stats.set(model, { model, commits: 0, testsAdded: 0, typeScriptErrors: 0, typeScriptClean: 0, hasAttribution: 0, commitsByType: {}, firstSeen: c.date, lastSeen: c.date, subjects: [], }); } const s = stats.get(model)!; s.commits++; if (c.agent) s.hasAttribution++; const added = parseTestsAdded(c.testsAdded); if (added >= 0) s.testsAdded += added; const tsErrors = parseTsErrors(c.typescript); if (tsErrors > 0) s.typeScriptErrors += tsErrors; if (tsErrors === 0) s.typeScriptClean++; const type = commitType(c.subject); s.commitsByType[type] = (s.commitsByType[type] || 0) + 1; if (c.date < s.firstSeen) s.firstSeen = c.date; if (c.date > s.lastSeen) s.lastSeen = c.date; s.subjects.push(`${c.hash} ${c.subject}`); } return stats; } function printTable(stats: Map, commits: CommitRecord[]) { const total = commits.length; const withAttribution = commits.filter(c => c.agent).length; console.log('\nšŸ“Š Recipe Manager — Model Performance Report'); console.log(` Generated: ${new Date().toISOString()}`); console.log(` Total commits: ${total} | With attribution: ${withAttribution} | Unknown: ${total - withAttribution}\n`); // Sort by commit count desc const sorted = [...stats.values()].sort((a, b) => b.commits - a.commits); const col = (s: string, w: number) => s.slice(0, w).padEnd(w); const header = [ col('Model', 36), col('Commits', 8), col('Tests+', 8), col('TS-clean', 9), col('TS-errors', 10), col('Last seen', 12), ].join(' │ '); const divider = '─'.repeat(header.length); console.log(divider); console.log(header); console.log(divider); for (const s of sorted) { const tsClean = s.typeScriptClean > 0 ? `${s.typeScriptClean}/${s.commits}` : '?'; const tsErrors = s.typeScriptErrors > 0 ? `āš ļø ${s.typeScriptErrors}` : s.typeScriptErrors === 0 ? 'āœ… 0' : '?'; console.log([ col(s.model, 36), col(String(s.commits), 8), col(s.testsAdded > 0 ? `+${s.testsAdded}` : '?', 8), col(tsClean, 9), col(tsErrors, 10), col(s.lastSeen, 12), ].join(' │ ')); } console.log(divider); // Commits without attribution const unknown = commits.filter(c => !c.agent); if (unknown.length > 0) { console.log(`\nāš ļø ${unknown.length} commits without Agent: trailer:`); for (const c of unknown.slice(0, 10)) { console.log(` ${c.hash} ${c.date} ${c.subject}`); } if (unknown.length > 10) console.log(` ... and ${unknown.length - 10} more`); } // Commits with TS errors const tsErrorCommits = commits.filter(c => parseTsErrors(c.typescript) > 0); if (tsErrorCommits.length > 0) { console.log(`\nšŸ”“ Commits with TypeScript errors:`); for (const c of tsErrorCommits) { console.log(` ${c.hash} ${c.date} [${c.agent || 'unknown'}] ${c.subject}`); } } // Zero tests added on feat commits const featNoTests = commits.filter(c => commitType(c.subject) === 'feat' && c.testsAdded !== '' && parseTestsAdded(c.testsAdded) === 0 ); if (featNoTests.length > 0) { console.log(`\n🟔 feat commits with Tests-Added: 0 (no new tests):`); for (const c of featNoTests) { console.log(` ${c.hash} ${c.date} [${c.agent || 'unknown'}] ${c.subject}`); } } console.log(''); } function main() { const { since, asJson } = parseArgs(); const commits = getCommits(since); const stats = buildStats(commits); if (asJson) { console.log(JSON.stringify({ generatedAt: new Date().toISOString(), totalCommits: commits.length, models: [...stats.values()], commits: commits, }, null, 2)); } else { printTable(stats, commits); } } main();