246 lines
6.9 KiB
TypeScript
246 lines
6.9 KiB
TypeScript
#!/usr/bin/env ts-node
|
||
/**
|
||
* scripts/model-report.ts
|
||
*
|
||
* Parses git log trailers to generate a model performance table.
|
||
* Reads: Agent, Tests, Tests-Added, TypeScript trailers from commit messages.
|
||
*
|
||
* Usage:
|
||
* npx ts-node scripts/model-report.ts
|
||
* npx ts-node scripts/model-report.ts --since=2026-03-01
|
||
* npx ts-node scripts/model-report.ts --json
|
||
*/
|
||
|
||
import { execSync } from 'child_process';
|
||
|
||
interface CommitRecord {
|
||
hash: string;
|
||
date: string;
|
||
subject: string;
|
||
agent: string;
|
||
tests: string; // raw e.g. "129/129 passing"
|
||
testsAdded: string; // raw e.g. "+32"
|
||
typescript: string; // raw e.g. "clean" or "2 errors"
|
||
body: string;
|
||
}
|
||
|
||
interface ModelStats {
|
||
model: string;
|
||
commits: number;
|
||
testsAdded: number;
|
||
typeScriptErrors: number; // errors introduced
|
||
typeScriptClean: number; // commits that were clean
|
||
hasAttribution: number; // commits with Agent: trailer
|
||
commitsByType: Record<string, number>;
|
||
firstSeen: string;
|
||
lastSeen: string;
|
||
subjects: string[];
|
||
}
|
||
|
||
function parseArgs() {
|
||
const args = process.argv.slice(2);
|
||
const since = args.find(a => a.startsWith('--since='))?.split('=')[1];
|
||
const asJson = args.includes('--json');
|
||
return { since, asJson };
|
||
}
|
||
|
||
function getCommits(since?: string): CommitRecord[] {
|
||
const sinceFlag = since ? `--since="${since}"` : '';
|
||
// Use %x00 as field separator, %x01 as record separator
|
||
const format = '%H%x00%ad%x00%s%x00%b%x01';
|
||
const cmd = `git log ${sinceFlag} --format="${format}" --date=short`;
|
||
|
||
let raw: string;
|
||
try {
|
||
raw = execSync(cmd, { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 });
|
||
} catch {
|
||
console.error('Failed to run git log');
|
||
process.exit(1);
|
||
}
|
||
|
||
const records = raw.split('\x01').filter(r => r.trim());
|
||
return records.map(record => {
|
||
const [hash, date, subject, body = ''] = record.split('\x00');
|
||
|
||
const trailer = (key: string) => {
|
||
const match = body.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
|
||
return match ? match[1].trim() : '';
|
||
};
|
||
|
||
return {
|
||
hash: (hash || '').trim().slice(0, 7),
|
||
date: (date || '').trim(),
|
||
subject: (subject || '').trim(),
|
||
agent: trailer('Agent'),
|
||
tests: trailer('Tests'),
|
||
testsAdded: trailer('Tests-Added'),
|
||
typescript: trailer('TypeScript'),
|
||
body: body.trim(),
|
||
};
|
||
}).filter(r => r.hash);
|
||
}
|
||
|
||
function parseTestsAdded(raw: string): number {
|
||
if (!raw) return -1; // -1 = unknown (no trailer)
|
||
const match = raw.match(/[+-]?(\d+)/);
|
||
return match ? parseInt(match[1], 10) : 0;
|
||
}
|
||
|
||
function parseTsErrors(raw: string): number {
|
||
if (!raw) return -1; // -1 = unknown
|
||
if (raw.toLowerCase().includes('clean')) return 0;
|
||
const match = raw.match(/(\d+)/);
|
||
return match ? parseInt(match[1], 10) : -1;
|
||
}
|
||
|
||
function commitType(subject: string): string {
|
||
const match = subject.match(/^(feat|fix|refactor|test|docs|chore|build|ci|perf)/);
|
||
return match ? match[1] : 'other';
|
||
}
|
||
|
||
function buildStats(commits: CommitRecord[]): Map<string, ModelStats> {
|
||
const stats = new Map<string, ModelStats>();
|
||
|
||
for (const c of commits) {
|
||
const model = c.agent || 'unknown';
|
||
|
||
if (!stats.has(model)) {
|
||
stats.set(model, {
|
||
model,
|
||
commits: 0,
|
||
testsAdded: 0,
|
||
typeScriptErrors: 0,
|
||
typeScriptClean: 0,
|
||
hasAttribution: 0,
|
||
commitsByType: {},
|
||
firstSeen: c.date,
|
||
lastSeen: c.date,
|
||
subjects: [],
|
||
});
|
||
}
|
||
|
||
const s = stats.get(model)!;
|
||
s.commits++;
|
||
|
||
if (c.agent) s.hasAttribution++;
|
||
|
||
const added = parseTestsAdded(c.testsAdded);
|
||
if (added >= 0) s.testsAdded += added;
|
||
|
||
const tsErrors = parseTsErrors(c.typescript);
|
||
if (tsErrors > 0) s.typeScriptErrors += tsErrors;
|
||
if (tsErrors === 0) s.typeScriptClean++;
|
||
|
||
const type = commitType(c.subject);
|
||
s.commitsByType[type] = (s.commitsByType[type] || 0) + 1;
|
||
|
||
if (c.date < s.firstSeen) s.firstSeen = c.date;
|
||
if (c.date > s.lastSeen) s.lastSeen = c.date;
|
||
|
||
s.subjects.push(`${c.hash} ${c.subject}`);
|
||
}
|
||
|
||
return stats;
|
||
}
|
||
|
||
function printTable(stats: Map<string, ModelStats>, commits: CommitRecord[]) {
|
||
const total = commits.length;
|
||
const withAttribution = commits.filter(c => c.agent).length;
|
||
|
||
console.log('\n📊 Recipe Manager — Model Performance Report');
|
||
console.log(` Generated: ${new Date().toISOString()}`);
|
||
console.log(` Total commits: ${total} | With attribution: ${withAttribution} | Unknown: ${total - withAttribution}\n`);
|
||
|
||
// Sort by commit count desc
|
||
const sorted = [...stats.values()].sort((a, b) => b.commits - a.commits);
|
||
|
||
const col = (s: string, w: number) => s.slice(0, w).padEnd(w);
|
||
|
||
const header = [
|
||
col('Model', 36),
|
||
col('Commits', 8),
|
||
col('Tests+', 8),
|
||
col('TS-clean', 9),
|
||
col('TS-errors', 10),
|
||
col('Last seen', 12),
|
||
].join(' │ ');
|
||
|
||
const divider = '─'.repeat(header.length);
|
||
console.log(divider);
|
||
console.log(header);
|
||
console.log(divider);
|
||
|
||
for (const s of sorted) {
|
||
const tsClean = s.typeScriptClean > 0
|
||
? `${s.typeScriptClean}/${s.commits}`
|
||
: '?';
|
||
const tsErrors = s.typeScriptErrors > 0
|
||
? `⚠️ ${s.typeScriptErrors}`
|
||
: s.typeScriptErrors === 0 ? '✅ 0' : '?';
|
||
|
||
console.log([
|
||
col(s.model, 36),
|
||
col(String(s.commits), 8),
|
||
col(s.testsAdded > 0 ? `+${s.testsAdded}` : '?', 8),
|
||
col(tsClean, 9),
|
||
col(tsErrors, 10),
|
||
col(s.lastSeen, 12),
|
||
].join(' │ '));
|
||
}
|
||
|
||
console.log(divider);
|
||
|
||
// Commits without attribution
|
||
const unknown = commits.filter(c => !c.agent);
|
||
if (unknown.length > 0) {
|
||
console.log(`\n⚠️ ${unknown.length} commits without Agent: trailer:`);
|
||
for (const c of unknown.slice(0, 10)) {
|
||
console.log(` ${c.hash} ${c.date} ${c.subject}`);
|
||
}
|
||
if (unknown.length > 10) console.log(` ... and ${unknown.length - 10} more`);
|
||
}
|
||
|
||
// Commits with TS errors
|
||
const tsErrorCommits = commits.filter(c => parseTsErrors(c.typescript) > 0);
|
||
if (tsErrorCommits.length > 0) {
|
||
console.log(`\n🔴 Commits with TypeScript errors:`);
|
||
for (const c of tsErrorCommits) {
|
||
console.log(` ${c.hash} ${c.date} [${c.agent || 'unknown'}] ${c.subject}`);
|
||
}
|
||
}
|
||
|
||
// Zero tests added on feat commits
|
||
const featNoTests = commits.filter(c =>
|
||
commitType(c.subject) === 'feat' &&
|
||
c.testsAdded !== '' &&
|
||
parseTestsAdded(c.testsAdded) === 0
|
||
);
|
||
if (featNoTests.length > 0) {
|
||
console.log(`\n🟡 feat commits with Tests-Added: 0 (no new tests):`);
|
||
for (const c of featNoTests) {
|
||
console.log(` ${c.hash} ${c.date} [${c.agent || 'unknown'}] ${c.subject}`);
|
||
}
|
||
}
|
||
|
||
console.log('');
|
||
}
|
||
|
||
function main() {
|
||
const { since, asJson } = parseArgs();
|
||
const commits = getCommits(since);
|
||
const stats = buildStats(commits);
|
||
|
||
if (asJson) {
|
||
console.log(JSON.stringify({
|
||
generatedAt: new Date().toISOString(),
|
||
totalCommits: commits.length,
|
||
models: [...stats.values()],
|
||
commits: commits,
|
||
}, null, 2));
|
||
} else {
|
||
printTable(stats, commits);
|
||
}
|
||
}
|
||
|
||
main();
|