fix(workflow): align orchestrator API and workflow runner
This commit is contained in:
parent
476ca0b0c2
commit
0510009597
|
|
@ -9,7 +9,8 @@
|
||||||
"build": "tsc",
|
"build": "tsc",
|
||||||
"test": "vitest run",
|
"test": "vitest run",
|
||||||
"test:watch": "vitest",
|
"test:watch": "vitest",
|
||||||
"migrate": "ts-node-esm src/backend/db/migrate.ts"
|
"migrate": "ts-node-esm src/backend/db/migrate.ts",
|
||||||
|
"workflow:run": "ts-node scripts/run-workflow.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"express": "^4.18.2",
|
"express": "^4.18.2",
|
||||||
|
|
|
||||||
|
|
@ -1,147 +1,103 @@
|
||||||
#!/usr/bin/env ts-node
|
|
||||||
import { WorkflowStatusManager } from '../src/backend/services/WorkflowStatusManager';
|
import { WorkflowStatusManager } from '../src/backend/services/WorkflowStatusManager';
|
||||||
import { getPendingPhaseUpdates } from '../src/backend/services/PhaseUpdateQueue';
|
import { getPendingPhaseUpdates } from '../src/backend/services/PhaseUpdateQueue';
|
||||||
import * as path from 'path';
|
import { exec as execCb } from 'child_process';
|
||||||
|
import { promisify } from 'util';
|
||||||
|
|
||||||
// Configurable recency window in hours for recent commits
|
const exec = promisify(execCb);
|
||||||
type ReportConfig = {
|
|
||||||
|
type CommitInfo = { hash: string; msg: string; date: string };
|
||||||
|
|
||||||
|
type MorningReportOptions = {
|
||||||
commitWindowHours?: number;
|
commitWindowHours?: number;
|
||||||
statusPath?: string;
|
|
||||||
phaseUpdatesPath?: string;
|
|
||||||
stalledThresholdMinutes?: number;
|
stalledThresholdMinutes?: number;
|
||||||
now?: Date;
|
now?: Date;
|
||||||
getRecentCommitsFn?: (windowHours: number) => Promise<{hash: string; msg: string; date: string}[]>;
|
getRecentCommitsFn?: (sinceIso: string) => Promise<CommitInfo[]>;
|
||||||
};
|
};
|
||||||
|
|
||||||
const DEFAULT_STATUS_PATH = path.join(process.cwd(), 'status/workflow-status.json');
|
async function getRecentCommits(sinceIso: string): Promise<CommitInfo[]> {
|
||||||
const DEFAULT_PHASE_UPDATES_PATH = path.join(process.cwd(), 'status/phase-updates.jsonl');
|
const cmd = `git log --since="${sinceIso}" --pretty=format:"%h|%s|%cI"`;
|
||||||
const DEFAULT_STALLED_THRESHOLD_MINUTES = 60;
|
const { stdout } = await exec(cmd);
|
||||||
|
if (!stdout.trim()) return [];
|
||||||
export async function getRecentCommits(windowHours: number): Promise<{hash: string, msg: string, date: string}[]> {
|
return stdout
|
||||||
const sinceArg = `--since='${windowHours} hours ago'`;
|
.split(/\r?\n/)
|
||||||
const cmd = `git log --oneline --date=iso --pretty=format:'%h|%s|%cd' ${sinceArg}`;
|
.map((line) => line.trim())
|
||||||
const { exec } = require('child_process');
|
.filter(Boolean)
|
||||||
return new Promise((resolve) => {
|
.map((line) => {
|
||||||
exec(cmd, { cwd: process.cwd() }, (err: any, stdout: string) => {
|
const [hash, msg, date] = line.split('|');
|
||||||
if (err) return resolve([]);
|
return { hash, msg, date };
|
||||||
const lines = stdout.trim().split(/\n/);
|
|
||||||
const result = lines.filter(Boolean).map((line: string) => {
|
|
||||||
const [hash, msg, date] = line.split('|');
|
|
||||||
return {hash, msg, date};
|
|
||||||
});
|
|
||||||
resolve(result);
|
|
||||||
});
|
});
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function minutesBetween(a: Date, b: Date): number {
|
function minutesSince(iso: string, now: Date): number {
|
||||||
return Math.abs((a.getTime() - b.getTime()) / 60000);
|
return Math.floor((now.getTime() - new Date(iso).getTime()) / 60000);
|
||||||
}
|
}
|
||||||
|
|
||||||
function detectStalled(status: any, thresholdMinutes: number, now: Date): {stalled: boolean; reason?: string; recommend?: string} {
|
export async function generateMorningReport(options: MorningReportOptions = {}): Promise<string> {
|
||||||
if (!status) return {stalled: false};
|
const commitWindowHours = options.commitWindowHours ?? 24;
|
||||||
if (["completed", "failed", "idle"].includes(status.overallStatus)) return {stalled: false};
|
const stalledThresholdMinutes = options.stalledThresholdMinutes ?? 60;
|
||||||
const last = new Date(status.lastUpdated);
|
const now = options.now ?? new Date();
|
||||||
const mins = minutesBetween(now, last);
|
const getRecentCommitsFn = options.getRecentCommitsFn ?? getRecentCommits;
|
||||||
if (mins >= thresholdMinutes) {
|
|
||||||
return {
|
|
||||||
stalled: true,
|
|
||||||
reason: `No progress in ${mins.toFixed(0)} minutes (threshold: ${thresholdMinutes}m). Last update: ${last.toISOString()}`,
|
|
||||||
recommend: status.overallStatus === "blocked" ? "Manual intervention may be required. See blockers below." : "Restart or debug orchestrator."
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return {stalled: false};
|
|
||||||
}
|
|
||||||
|
|
||||||
function extractBlockers(status: any): string[] {
|
const since = new Date(now.getTime() - commitWindowHours * 60 * 60 * 1000).toISOString();
|
||||||
const out: string[] = [];
|
const commits = await getRecentCommitsFn(since);
|
||||||
if (!status) return out;
|
|
||||||
if (status.overallStatus === 'blocked') {
|
|
||||||
if (status.lastFailureReason) out.push(`❗ Blocked: ${status.lastFailureReason}`);
|
|
||||||
else out.push(`❗ Blocked: Reason not specified.`);
|
|
||||||
}
|
|
||||||
if (status.overallStatus === 'failed') {
|
|
||||||
out.push('❌ Workflow failed.');
|
|
||||||
if (status.lastFailureReason) out.push(`Failure: ${status.lastFailureReason}`);
|
|
||||||
}
|
|
||||||
if (status.nextAction && status.nextAction.includes('manual')) {
|
|
||||||
out.push(`⚠️ Manual intervention required: ${status.nextAction}`);
|
|
||||||
}
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function main(config: ReportConfig = {}) {
|
const statusManager = new WorkflowStatusManager();
|
||||||
const commitWindow = config.commitWindowHours || 24;
|
const status = await statusManager.read();
|
||||||
const statusPath = config.statusPath || DEFAULT_STATUS_PATH;
|
const pending = await getPendingPhaseUpdates();
|
||||||
const phaseUpdatesPath = config.phaseUpdatesPath || DEFAULT_PHASE_UPDATES_PATH;
|
|
||||||
const threshold = config.stalledThresholdMinutes || DEFAULT_STALLED_THRESHOLD_MINUTES;
|
|
||||||
const now = config.now || new Date();
|
|
||||||
|
|
||||||
// 1. Recent commits
|
let out = '# 🌅 Morning Workflow Report\n\n';
|
||||||
const commits = await (config.getRecentCommitsFn || getRecentCommits)(commitWindow);
|
out += `## Recent Commits (last ${commitWindowHours}h)\n`;
|
||||||
|
if (!commits.length) {
|
||||||
// 2. Workflow status
|
out += '- (No commits in window)\n\n';
|
||||||
const wsm = new WorkflowStatusManager(statusPath);
|
} else {
|
||||||
const status = await wsm.read();
|
|
||||||
|
|
||||||
// 3. Blockers
|
|
||||||
const blockers = extractBlockers(status);
|
|
||||||
|
|
||||||
// 4. Pending phase updates
|
|
||||||
const pendingUpdates = await getPendingPhaseUpdates();
|
|
||||||
|
|
||||||
// 5. Stalled-state detection
|
|
||||||
const stalled = detectStalled(status, threshold, now);
|
|
||||||
|
|
||||||
// Render report
|
|
||||||
let out = `# 🌅 Morning Workflow Report\n`;
|
|
||||||
|
|
||||||
out += `\n## Recent Commits (last ${commitWindow}h)\n`;
|
|
||||||
if (commits.length > 0) {
|
|
||||||
for (const c of commits) {
|
for (const c of commits) {
|
||||||
out += `- \`${c.hash}\` ${c.msg} _(at ${c.date})_\n`;
|
out += `- \`${c.hash}\` ${c.msg} _(at ${c.date})_\n`;
|
||||||
}
|
}
|
||||||
|
out += '\n';
|
||||||
|
}
|
||||||
|
|
||||||
|
out += '## Workflow Status\n';
|
||||||
|
if (!status) {
|
||||||
|
out += '- No workflow status available\n\n';
|
||||||
} else {
|
} else {
|
||||||
out += '(No commits in window)\n';
|
out += `- Current phase: **${status.currentPhase ?? '-'}**\n`;
|
||||||
}
|
out += `- State: **${status.overallStatus ?? '-'}**\n`;
|
||||||
|
out += `- Last updated: ${status.lastUpdated ?? '-'}\n`;
|
||||||
|
out += `- Completed phases: ${(status.completedPhases ?? []).join(', ') || '-'}\n\n`;
|
||||||
|
|
||||||
out += `\n## Workflow Status\n`;
|
if (status.overallStatus === 'running' && status.lastUpdated) {
|
||||||
if (status) {
|
const elapsed = minutesSince(status.lastUpdated, now);
|
||||||
out += `- Current phase: **${status.currentPhase || '—'}**\n`;
|
if (elapsed >= stalledThresholdMinutes) {
|
||||||
out += `- State: **${status.overallStatus}**\n`;
|
out += '## ⚠️ Workflow Stalled\n';
|
||||||
out += `- Last updated: ${status.lastUpdated}\n`;
|
out += `- Reason: No progress in ${elapsed} minutes (threshold: ${stalledThresholdMinutes}m). Last update: ${status.lastUpdated}\n`;
|
||||||
if (status.completedPhases && status.completedPhases.length)
|
out += '- Recommendation: Restart or debug orchestrator.\n\n';
|
||||||
out += `- Completed phases: ${status.completedPhases.join(', ')}\n`;
|
}
|
||||||
} else {
|
}
|
||||||
out += '- No workflow status available.\n';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (stalled.stalled) {
|
if (status.lastFailureReason) {
|
||||||
out += `\n## ⚠️ Workflow Stalled\n- Reason: ${stalled.reason}\n- Recommendation: ${stalled.recommend}\n`;
|
out += '## Blockers\n';
|
||||||
}
|
out += `- ❗ ${status.lastFailureReason}\n\n`;
|
||||||
|
|
||||||
if (blockers.length > 0) {
|
|
||||||
out += `\n## Blockers\n`;
|
|
||||||
for (const b of blockers) {
|
|
||||||
out += `- ${b}\n`;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
out += `\n## Pending Phase Updates\n`;
|
out += '## Pending Phase Updates\n';
|
||||||
if (pendingUpdates.length > 0) {
|
if (!pending.length) {
|
||||||
for (const e of pendingUpdates) {
|
out += '- All phase updates relayed\n';
|
||||||
out += `- [${e.eventType}] ${e.summary} (phase: ${e.phase}) at ${e.timestamp} [id: ${e.id}]\n`;
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
out += '(All phase updates relayed)\n';
|
for (const p of pending) {
|
||||||
|
out += `- [${p.eventType}] ${p.summary} (phase: ${p.phase ?? '-'}) at ${p.timestamp} [id: ${p.id}]\n`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(out);
|
console.log(out);
|
||||||
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
// CLI wrapper
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||||
if (require.main === module) {
|
generateMorningReport().catch((err) => {
|
||||||
main();
|
console.error('Failed to generate morning report', err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export default main;
|
export default generateMorningReport;
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,190 @@
|
||||||
|
#!/usr/bin/env ts-node
|
||||||
|
import * as path from 'path';
|
||||||
|
import { promises as fs } from 'fs';
|
||||||
|
import { SequentialOrchestrator } from '../src/backend/services/SequentialOrchestrator.ts';
|
||||||
|
import type { WorkflowStage, WorkflowContext } from '../src/backend/services/SequentialOrchestrator.ts';
|
||||||
|
import { WorkflowStatusManager } from '../src/backend/services/WorkflowStatusManager.ts';
|
||||||
|
|
||||||
|
type RunWorkflowOptions = {
|
||||||
|
mode?: 'start' | 'resume' | 'restart';
|
||||||
|
checkpointPath?: string;
|
||||||
|
statusPath?: string;
|
||||||
|
progressLogPath?: string;
|
||||||
|
maxRetriesPerPhase?: number;
|
||||||
|
now?: Date;
|
||||||
|
};
|
||||||
|
|
||||||
|
type CliArgs = {
|
||||||
|
mode: 'start' | 'resume' | 'restart';
|
||||||
|
checkpointPath?: string;
|
||||||
|
statusPath?: string;
|
||||||
|
progressLogPath?: string;
|
||||||
|
maxRetriesPerPhase?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
const DEFAULT_CHECKPOINT_PATH = path.join(process.cwd(), 'status', 'workflow-checkpoint.json');
|
||||||
|
const DEFAULT_STATUS_PATH = path.join(process.cwd(), 'status', 'workflow-status.json');
|
||||||
|
const DEFAULT_PROGRESS_LOG_PATH = path.join(process.cwd(), 'status', 'workflow-progress.jsonl');
|
||||||
|
|
||||||
|
function parseCliArgs(argv: string[]): CliArgs {
|
||||||
|
const out: CliArgs = { mode: 'resume' };
|
||||||
|
|
||||||
|
for (let i = 0; i < argv.length; i += 1) {
|
||||||
|
const arg = argv[i];
|
||||||
|
const next = argv[i + 1];
|
||||||
|
|
||||||
|
if (arg === '--mode' && next) {
|
||||||
|
if (next === 'start' || next === 'resume' || next === 'restart') {
|
||||||
|
out.mode = next;
|
||||||
|
i += 1;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (arg === '--checkpoint' && next) {
|
||||||
|
out.checkpointPath = next;
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (arg === '--status' && next) {
|
||||||
|
out.statusPath = next;
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (arg === '--progress-log' && next) {
|
||||||
|
out.progressLogPath = next;
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (arg === '--max-retries' && next) {
|
||||||
|
const parsed = Number.parseInt(next, 10);
|
||||||
|
if (Number.isFinite(parsed) && parsed >= 0) {
|
||||||
|
out.maxRetriesPerPhase = parsed;
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createTimestampedSummary(label: string, now: Date): string {
|
||||||
|
return `${label} @ ${now.toISOString()}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createStages(now: Date): WorkflowStage[] {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
id: 'review-plan',
|
||||||
|
name: 'Review TODO and roadmap',
|
||||||
|
execute: async (_context: WorkflowContext) => {
|
||||||
|
const todoPath = path.join(process.cwd(), 'TODO.md');
|
||||||
|
const roadmapPath = path.join(process.cwd(), 'ROADMAP.md');
|
||||||
|
const todo = await fs.readFile(todoPath, 'utf8');
|
||||||
|
const roadmap = await fs.readFile(roadmapPath, 'utf8');
|
||||||
|
const todoLines = todo.split(/\r?\n/).length;
|
||||||
|
const roadmapLines = roadmap.split(/\r?\n/).length;
|
||||||
|
return {
|
||||||
|
summary: createTimestampedSummary(`Loaded planning docs (TODO: ${todoLines} lines, ROADMAP: ${roadmapLines} lines)`, now),
|
||||||
|
metadata: { todoLines, roadmapLines },
|
||||||
|
nextAction: 'Proceed to implementation stage.',
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'verify-build',
|
||||||
|
name: 'Verify TypeScript build',
|
||||||
|
execute: async (_context: WorkflowContext) => {
|
||||||
|
// Lightweight validation step to keep workflow deterministic in tests.
|
||||||
|
const packageJsonPath = path.join(process.cwd(), 'package.json');
|
||||||
|
const pkgRaw = await fs.readFile(packageJsonPath, 'utf8');
|
||||||
|
const pkg = JSON.parse(pkgRaw) as { scripts?: Record<string, string> };
|
||||||
|
const hasBuildScript = Boolean(pkg.scripts?.build);
|
||||||
|
|
||||||
|
if (!hasBuildScript) {
|
||||||
|
throw new Error('Missing required npm script: build');
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
summary: createTimestampedSummary('Validated project scripts and build entrypoint', now),
|
||||||
|
metadata: { hasBuildScript },
|
||||||
|
nextAction: 'Proceed to tests stage.',
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'verify-tests',
|
||||||
|
name: 'Verify test command availability',
|
||||||
|
execute: async (_context: WorkflowContext) => {
|
||||||
|
const packageJsonPath = path.join(process.cwd(), 'package.json');
|
||||||
|
const pkgRaw = await fs.readFile(packageJsonPath, 'utf8');
|
||||||
|
const pkg = JSON.parse(pkgRaw) as { scripts?: Record<string, string> };
|
||||||
|
const hasTestScript = Boolean(pkg.scripts?.test);
|
||||||
|
|
||||||
|
if (!hasTestScript) {
|
||||||
|
throw new Error('Missing required npm script: test');
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
summary: createTimestampedSummary('Validated test script presence', now),
|
||||||
|
metadata: { hasTestScript },
|
||||||
|
nextAction: 'Workflow checks complete.',
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runWorkflow(options: RunWorkflowOptions = {}): Promise<void> {
|
||||||
|
const checkpointPath = options.checkpointPath ?? DEFAULT_CHECKPOINT_PATH;
|
||||||
|
const statusPath = options.statusPath ?? DEFAULT_STATUS_PATH;
|
||||||
|
const progressLogPath = options.progressLogPath ?? DEFAULT_PROGRESS_LOG_PATH;
|
||||||
|
const mode = options.mode ?? 'resume';
|
||||||
|
const maxRetriesPerPhase = options.maxRetriesPerPhase ?? 1;
|
||||||
|
const now = options.now ?? new Date();
|
||||||
|
|
||||||
|
const workflow = new SequentialOrchestrator({
|
||||||
|
checkpointPath,
|
||||||
|
maxRetriesPerPhase,
|
||||||
|
statusManager: new WorkflowStatusManager(statusPath),
|
||||||
|
progressLogPath,
|
||||||
|
});
|
||||||
|
|
||||||
|
const initialContext: WorkflowContext = {
|
||||||
|
runId: `workflow-${now.toISOString()}`,
|
||||||
|
startedAt: now.toISOString(),
|
||||||
|
metadata: {
|
||||||
|
initiatedBy: 'scripts/run-workflow.ts',
|
||||||
|
mode,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const stages = createStages(now);
|
||||||
|
|
||||||
|
if (mode === 'restart' || mode === 'start') {
|
||||||
|
await workflow.start(stages, initialContext);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await workflow.resume(stages, initialContext);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||||
|
const args = parseCliArgs(process.argv.slice(2));
|
||||||
|
|
||||||
|
runWorkflow({
|
||||||
|
mode: args.mode,
|
||||||
|
checkpointPath: args.checkpointPath,
|
||||||
|
statusPath: args.statusPath,
|
||||||
|
progressLogPath: args.progressLogPath,
|
||||||
|
maxRetriesPerPhase: args.maxRetriesPerPhase,
|
||||||
|
}).catch((error) => {
|
||||||
|
const message = error instanceof Error ? error.message : String(error);
|
||||||
|
console.error(`[workflow:run] failed: ${message}`);
|
||||||
|
process.exitCode = 1;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
@ -12,17 +12,17 @@ export type PhaseProgressLogEntry = {
|
||||||
|
|
||||||
const PHASE_PROGRESS_LOG = path.join(process.cwd(), 'status/phase-progress.jsonl');
|
const PHASE_PROGRESS_LOG = path.join(process.cwd(), 'status/phase-progress.jsonl');
|
||||||
|
|
||||||
export async function logPhaseProgress(entry: PhaseProgressLogEntry): Promise<void> {
|
export async function logPhaseProgress(entry: PhaseProgressLogEntry, logPath = PHASE_PROGRESS_LOG): Promise<void> {
|
||||||
await fs.mkdir(path.dirname(PHASE_PROGRESS_LOG), { recursive: true });
|
await fs.mkdir(path.dirname(logPath), { recursive: true });
|
||||||
await fs.appendFile(PHASE_PROGRESS_LOG, JSON.stringify(entry) + '\n', 'utf8');
|
await fs.appendFile(logPath, JSON.stringify(entry) + '\n', 'utf8');
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getRecentPhaseProgress(limit = 20): Promise<PhaseProgressLogEntry[]> {
|
export async function getRecentPhaseProgress(limit = 20, logPath = PHASE_PROGRESS_LOG): Promise<PhaseProgressLogEntry[]> {
|
||||||
try {
|
try {
|
||||||
const data = await fs.readFile(PHASE_PROGRESS_LOG, 'utf8');
|
const data = await fs.readFile(logPath, 'utf8');
|
||||||
const lines = data.trim().split(/\r?\n/).filter(Boolean);
|
const lines = data.trim().split(/\r?\n/).filter(Boolean);
|
||||||
return lines.slice(-limit).map(line => JSON.parse(line));
|
return lines.slice(-limit).map((line) => JSON.parse(line));
|
||||||
} catch (err) {
|
} catch {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,30 @@
|
||||||
import { promises as fs } from 'fs';
|
import { promises as fs } from 'fs';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
import { logPhaseProgress } from './PhaseProgressLogger';
|
import { logPhaseProgress } from './PhaseProgressLogger.ts';
|
||||||
import { WorkflowStatusManager, WorkflowStatus } from './WorkflowStatusManager';
|
import { WorkflowStatusManager } from './WorkflowStatusManager.ts';
|
||||||
import { appendPhaseUpdate } from './PhaseUpdateQueue';
|
import type { WorkflowStatus } from './WorkflowStatusManager.ts';
|
||||||
|
import { appendPhaseUpdate } from './PhaseUpdateQueue.ts';
|
||||||
|
|
||||||
|
export type WorkflowContext = {
|
||||||
|
runId: string;
|
||||||
|
startedAt: string;
|
||||||
|
metadata?: Record<string, unknown>;
|
||||||
|
[key: string]: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type WorkflowStageResult = {
|
||||||
|
summary?: string;
|
||||||
|
metadata?: Record<string, unknown>;
|
||||||
|
nextAction?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type WorkflowStage = {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
execute: (context: WorkflowContext) => Promise<WorkflowStageResult | void>;
|
||||||
|
retry?: number;
|
||||||
|
backoffMs?: number;
|
||||||
|
};
|
||||||
|
|
||||||
export type OrchestratorPhase<TInput, TOutput> = {
|
export type OrchestratorPhase<TInput, TOutput> = {
|
||||||
name: string;
|
name: string;
|
||||||
|
|
@ -11,6 +33,7 @@ export type OrchestratorPhase<TInput, TOutput> = {
|
||||||
backoffMs?: number;
|
backoffMs?: number;
|
||||||
nextAction?: string;
|
nextAction?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type PhaseAttemptResult = {
|
export type PhaseAttemptResult = {
|
||||||
phase: string;
|
phase: string;
|
||||||
status: 'success' | 'failure';
|
status: 'success' | 'failure';
|
||||||
|
|
@ -20,58 +43,72 @@ export type PhaseAttemptResult = {
|
||||||
failureReason?: string;
|
failureReason?: string;
|
||||||
nextAction: string;
|
nextAction: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type Checkpoint = {
|
export type Checkpoint = {
|
||||||
currentPhase: number;
|
currentPhase: number;
|
||||||
phaseResults: PhaseAttemptResult[];
|
phaseResults: PhaseAttemptResult[];
|
||||||
inProgress?: boolean;
|
inProgress?: boolean;
|
||||||
|
context?: WorkflowContext;
|
||||||
};
|
};
|
||||||
|
|
||||||
const DEFAULT_CHECKPOINT_FILE = path.join(process.cwd(), 'data/orchestrator-checkpoint.json');
|
const DEFAULT_CHECKPOINT_FILE = path.join(process.cwd(), 'data/orchestrator-checkpoint.json');
|
||||||
const DEFAULT_STATUS_FILE = path.join(process.cwd(), 'status/workflow-status.json');
|
const DEFAULT_STATUS_FILE = path.join(process.cwd(), 'status/workflow-status.json');
|
||||||
|
const DEFAULT_PROGRESS_LOG = path.join(process.cwd(), 'status/phase-progress.jsonl');
|
||||||
|
|
||||||
export class SequentialOrchestrator<TInput = any, TOutput = any> {
|
export class SequentialOrchestrator<TInput = any, TOutput = any> {
|
||||||
private phases: OrchestratorPhase<TInput, TOutput>[];
|
private phases?: OrchestratorPhase<TInput, TOutput>[];
|
||||||
|
private input?: TInput;
|
||||||
private checkpointPath: string;
|
private checkpointPath: string;
|
||||||
private input: TInput;
|
|
||||||
private maxAttemptsPerPhasePerRun: number;
|
private maxAttemptsPerPhasePerRun: number;
|
||||||
|
private maxRetriesPerPhase?: number;
|
||||||
private statusManager: WorkflowStatusManager;
|
private statusManager: WorkflowStatusManager;
|
||||||
|
private progressLogPath: string;
|
||||||
|
|
||||||
constructor(options: {
|
constructor(options: {
|
||||||
phases: OrchestratorPhase<TInput, TOutput>[];
|
phases?: OrchestratorPhase<TInput, TOutput>[];
|
||||||
checkpointPath?: string;
|
checkpointPath?: string;
|
||||||
input: TInput;
|
input?: TInput;
|
||||||
maxAttemptsPerPhasePerRun?: number;
|
maxAttemptsPerPhasePerRun?: number;
|
||||||
|
maxRetriesPerPhase?: number;
|
||||||
|
statusManager?: WorkflowStatusManager;
|
||||||
statusFilePath?: string;
|
statusFilePath?: string;
|
||||||
}) {
|
progressLogPath?: string;
|
||||||
|
} = {}) {
|
||||||
this.phases = options.phases;
|
this.phases = options.phases;
|
||||||
this.checkpointPath = options.checkpointPath || DEFAULT_CHECKPOINT_FILE;
|
|
||||||
this.input = options.input;
|
this.input = options.input;
|
||||||
|
this.checkpointPath = options.checkpointPath || DEFAULT_CHECKPOINT_FILE;
|
||||||
this.maxAttemptsPerPhasePerRun = options.maxAttemptsPerPhasePerRun ?? Infinity;
|
this.maxAttemptsPerPhasePerRun = options.maxAttemptsPerPhasePerRun ?? Infinity;
|
||||||
this.statusManager = new WorkflowStatusManager(options.statusFilePath || DEFAULT_STATUS_FILE);
|
this.maxRetriesPerPhase = options.maxRetriesPerPhase;
|
||||||
|
this.statusManager = options.statusManager ?? new WorkflowStatusManager(options.statusFilePath || DEFAULT_STATUS_FILE);
|
||||||
|
this.progressLogPath = options.progressLogPath ?? DEFAULT_PROGRESS_LOG;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async writeStatus(
|
private async writeStatus(
|
||||||
|
stages: WorkflowStage[],
|
||||||
state: Partial<WorkflowStatus>,
|
state: Partial<WorkflowStatus>,
|
||||||
extra: { currentPhaseIdx?: number; phaseResult?: PhaseAttemptResult } = {}
|
checkpoint: Checkpoint,
|
||||||
|
extra: { currentPhaseIdx?: number | null; phaseResult?: PhaseAttemptResult } = {}
|
||||||
) {
|
) {
|
||||||
const checkpoint = await this.loadCheckpoint();
|
const completedPhases = stages
|
||||||
let completedPhases: string[] = [];
|
.slice(0, checkpoint.currentPhase)
|
||||||
if (checkpoint) {
|
.map((p) => p.name)
|
||||||
completedPhases = this.phases
|
.filter((phaseName) => checkpoint.phaseResults.some((r) => r.phase === phaseName && r.status === 'success'));
|
||||||
.slice(0, checkpoint.currentPhase)
|
|
||||||
.map(p => p.name)
|
const currentP = typeof extra.currentPhaseIdx === 'number' && stages[extra.currentPhaseIdx]?.name
|
||||||
.filter(phaseName => checkpoint.phaseResults.some(r => r.phase === phaseName && r.status === 'success'));
|
? stages[extra.currentPhaseIdx].name
|
||||||
}
|
|
||||||
const currentP = typeof extra.currentPhaseIdx === 'number' && this.phases[extra.currentPhaseIdx]?.name
|
|
||||||
? this.phases[extra.currentPhaseIdx].name
|
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
let overallStatus = state.overallStatus || 'idle';
|
let overallStatus = state.overallStatus || 'idle';
|
||||||
if (extra.phaseResult && extra.phaseResult.status === 'failure') {
|
if (extra.phaseResult && extra.phaseResult.status === 'failure') {
|
||||||
overallStatus = 'failed';
|
overallStatus = 'failed';
|
||||||
} else if (checkpoint && checkpoint.currentPhase === this.phases.length) {
|
} else if (checkpoint.currentPhase === stages.length) {
|
||||||
overallStatus = 'completed';
|
overallStatus = 'completed';
|
||||||
}
|
}
|
||||||
|
|
||||||
const lastFailureReason = (state.lastFailureReason === undefined && extra.phaseResult?.failureReason)
|
const lastFailureReason = (state.lastFailureReason === undefined && extra.phaseResult?.failureReason)
|
||||||
? extra.phaseResult?.failureReason || null
|
? extra.phaseResult.failureReason || null
|
||||||
: (state.lastFailureReason !== undefined ? state.lastFailureReason : null);
|
: (state.lastFailureReason !== undefined ? state.lastFailureReason : null);
|
||||||
|
|
||||||
const workflowStatus: WorkflowStatus = {
|
const workflowStatus: WorkflowStatus = {
|
||||||
currentPhase: currentP,
|
currentPhase: currentP,
|
||||||
overallStatus,
|
overallStatus,
|
||||||
|
|
@ -80,8 +117,10 @@ export class SequentialOrchestrator<TInput = any, TOutput = any> {
|
||||||
nextAction: state.nextAction || extra.phaseResult?.nextAction || '',
|
nextAction: state.nextAction || extra.phaseResult?.nextAction || '',
|
||||||
completedPhases,
|
completedPhases,
|
||||||
};
|
};
|
||||||
|
|
||||||
await this.statusManager.update(workflowStatus);
|
await this.statusManager.update(workflowStatus);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async loadCheckpoint(): Promise<Checkpoint | null> {
|
private async loadCheckpoint(): Promise<Checkpoint | null> {
|
||||||
try {
|
try {
|
||||||
const txt = await fs.readFile(this.checkpointPath, 'utf8');
|
const txt = await fs.readFile(this.checkpointPath, 'utf8');
|
||||||
|
|
@ -90,72 +129,107 @@ export class SequentialOrchestrator<TInput = any, TOutput = any> {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async saveCheckpoint(checkpoint: Checkpoint) {
|
private async saveCheckpoint(checkpoint: Checkpoint) {
|
||||||
await fs.mkdir(path.dirname(this.checkpointPath), { recursive: true });
|
await fs.mkdir(path.dirname(this.checkpointPath), { recursive: true });
|
||||||
await fs.writeFile(this.checkpointPath, JSON.stringify(checkpoint, null, 2), 'utf8');
|
await fs.writeFile(this.checkpointPath, JSON.stringify(checkpoint, null, 2), 'utf8');
|
||||||
}
|
}
|
||||||
public async run(): Promise<void> {
|
|
||||||
|
private toStagesFromLegacyPhases(phases: OrchestratorPhase<TInput, TOutput>[], input: TInput): WorkflowStage[] {
|
||||||
|
return phases.map((phase, idx) => ({
|
||||||
|
id: `legacy-${idx + 1}`,
|
||||||
|
name: phase.name,
|
||||||
|
retry: phase.retry,
|
||||||
|
backoffMs: phase.backoffMs,
|
||||||
|
execute: async () => {
|
||||||
|
await phase.run(input);
|
||||||
|
return { nextAction: phase.nextAction ?? 'proceed' };
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
private async executeStages(stages: WorkflowStage[], context: WorkflowContext, resetCheckpoint: boolean): Promise<void> {
|
||||||
let checkpoint = await this.loadCheckpoint();
|
let checkpoint = await this.loadCheckpoint();
|
||||||
if (!checkpoint) {
|
|
||||||
checkpoint = { currentPhase: 0, phaseResults: [] };
|
if (!checkpoint || resetCheckpoint) {
|
||||||
|
checkpoint = { currentPhase: 0, phaseResults: [], inProgress: true, context };
|
||||||
|
await this.saveCheckpoint(checkpoint);
|
||||||
|
await this.writeStatus(stages, { overallStatus: 'idle' }, checkpoint, { currentPhaseIdx: 0 });
|
||||||
|
} else if (!checkpoint.context) {
|
||||||
|
checkpoint.context = context;
|
||||||
await this.saveCheckpoint(checkpoint);
|
await this.saveCheckpoint(checkpoint);
|
||||||
await this.writeStatus({ overallStatus: 'idle' }, { currentPhaseIdx: 0 });
|
|
||||||
}
|
}
|
||||||
while (checkpoint.currentPhase < this.phases.length) {
|
|
||||||
const phase = this.phases[checkpoint.currentPhase];
|
while (checkpoint.currentPhase < stages.length) {
|
||||||
|
const stage = stages[checkpoint.currentPhase];
|
||||||
let success = false;
|
let success = false;
|
||||||
let attempt = 1;
|
let attempt = 1;
|
||||||
const maxRetries = (typeof phase.retry === 'number') ? phase.retry : 1;
|
const stageRetries = typeof stage.retry === 'number' ? stage.retry : (this.maxRetriesPerPhase ?? 1);
|
||||||
const maxAttemptsThisRun = Number.isFinite(this.maxAttemptsPerPhasePerRun)
|
const maxAttemptsThisRun = Number.isFinite(this.maxAttemptsPerPhasePerRun)
|
||||||
? Math.min(maxRetries, this.maxAttemptsPerPhasePerRun)
|
? Math.min(stageRetries, this.maxAttemptsPerPhasePerRun)
|
||||||
: maxRetries;
|
: stageRetries;
|
||||||
|
|
||||||
await appendPhaseUpdate({
|
await appendPhaseUpdate({
|
||||||
eventType: 'phase_started',
|
eventType: 'phase_started',
|
||||||
phase: phase.name,
|
phase: stage.name,
|
||||||
summary: `Phase '${phase.name}' started`,
|
summary: `Phase '${stage.name}' started`,
|
||||||
details: { attempt },
|
details: { attempt },
|
||||||
});
|
});
|
||||||
for (; attempt <= maxAttemptsThisRun && !success; attempt++) {
|
|
||||||
|
for (; attempt <= maxAttemptsThisRun && !success; attempt += 1) {
|
||||||
try {
|
try {
|
||||||
await phase.run(this.input);
|
const result = await stage.execute(checkpoint.context ?? context);
|
||||||
|
const nextAction = result?.nextAction ?? 'proceed';
|
||||||
const res: PhaseAttemptResult = {
|
const res: PhaseAttemptResult = {
|
||||||
phase: phase.name,
|
phase: stage.name,
|
||||||
status: 'success',
|
status: 'success',
|
||||||
attempt,
|
attempt,
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
nextAction: 'proceed',
|
nextAction,
|
||||||
};
|
};
|
||||||
|
|
||||||
checkpoint.phaseResults.push(res);
|
checkpoint.phaseResults.push(res);
|
||||||
checkpoint.currentPhase += 1;
|
checkpoint.currentPhase += 1;
|
||||||
|
checkpoint.inProgress = checkpoint.currentPhase < stages.length;
|
||||||
await this.saveCheckpoint(checkpoint);
|
await this.saveCheckpoint(checkpoint);
|
||||||
|
|
||||||
await logPhaseProgress({
|
await logPhaseProgress({
|
||||||
phase: res.phase,
|
phase: res.phase,
|
||||||
status: res.status,
|
status: res.status,
|
||||||
attempt: res.attempt,
|
attempt: res.attempt,
|
||||||
timestamp: res.timestamp,
|
timestamp: res.timestamp,
|
||||||
nextAction: res.nextAction,
|
nextAction: res.nextAction,
|
||||||
});
|
}, this.progressLogPath);
|
||||||
|
|
||||||
await appendPhaseUpdate({
|
await appendPhaseUpdate({
|
||||||
eventType: 'phase_succeeded',
|
eventType: 'phase_succeeded',
|
||||||
phase: phase.name,
|
phase: stage.name,
|
||||||
summary: `Phase '${phase.name}' succeeded`,
|
summary: `Phase '${stage.name}' succeeded`,
|
||||||
details: { attempt },
|
details: { attempt },
|
||||||
});
|
});
|
||||||
|
|
||||||
success = true;
|
success = true;
|
||||||
await this.writeStatus({ overallStatus: 'running', nextAction: phase.nextAction || '' }, { currentPhaseIdx: checkpoint.currentPhase, phaseResult: res });
|
await this.writeStatus(
|
||||||
|
stages,
|
||||||
|
{ overallStatus: checkpoint.currentPhase < stages.length ? 'running' : 'completed', nextAction },
|
||||||
|
checkpoint,
|
||||||
|
{ currentPhaseIdx: checkpoint.currentPhase, phaseResult: res }
|
||||||
|
);
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
const isLastAttempt = attempt === maxRetries;
|
const isLastAttempt = attempt === stageRetries;
|
||||||
// Always log every failed attempt
|
const message = err && err.message ? err.message : String(err);
|
||||||
const failRes: PhaseAttemptResult = {
|
const failRes: PhaseAttemptResult = {
|
||||||
phase: phase.name,
|
phase: stage.name,
|
||||||
status: 'failure',
|
status: 'failure',
|
||||||
attempt,
|
attempt,
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
error: err && err.message ? err.message : String(err),
|
error: message,
|
||||||
failureReason: err && err.message ? err.message : String(err),
|
failureReason: message,
|
||||||
nextAction: !isLastAttempt ? 'retry' : 'manual intervention',
|
nextAction: !isLastAttempt ? 'retry' : 'manual intervention',
|
||||||
};
|
};
|
||||||
|
|
||||||
checkpoint.phaseResults.push(failRes);
|
checkpoint.phaseResults.push(failRes);
|
||||||
|
|
||||||
await logPhaseProgress({
|
await logPhaseProgress({
|
||||||
phase: failRes.phase,
|
phase: failRes.phase,
|
||||||
status: failRes.status,
|
status: failRes.status,
|
||||||
|
|
@ -163,30 +237,62 @@ export class SequentialOrchestrator<TInput = any, TOutput = any> {
|
||||||
timestamp: failRes.timestamp,
|
timestamp: failRes.timestamp,
|
||||||
failureReason: failRes.failureReason,
|
failureReason: failRes.failureReason,
|
||||||
nextAction: failRes.nextAction,
|
nextAction: failRes.nextAction,
|
||||||
});
|
}, this.progressLogPath);
|
||||||
|
|
||||||
|
if (!isLastAttempt && stage.backoffMs && stage.backoffMs > 0) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, stage.backoffMs));
|
||||||
|
}
|
||||||
|
|
||||||
if (isLastAttempt) {
|
if (isLastAttempt) {
|
||||||
|
checkpoint.inProgress = false;
|
||||||
await appendPhaseUpdate({
|
await appendPhaseUpdate({
|
||||||
eventType: 'phase_failed',
|
eventType: 'phase_failed',
|
||||||
phase: phase.name,
|
phase: stage.name,
|
||||||
summary: `Phase '${phase.name}' failed`,
|
summary: `Phase '${stage.name}' failed`,
|
||||||
details: { attempt, error: err && err.message ? err.message : String(err) },
|
details: { attempt, error: message },
|
||||||
});
|
});
|
||||||
await this.saveCheckpoint(checkpoint);
|
await this.saveCheckpoint(checkpoint);
|
||||||
await this.writeStatus({ overallStatus: 'failed', lastFailureReason: failRes.failureReason, nextAction: failRes.nextAction }, { currentPhaseIdx: checkpoint.currentPhase, phaseResult: failRes });
|
await this.writeStatus(
|
||||||
|
stages,
|
||||||
|
{ overallStatus: 'failed', lastFailureReason: failRes.failureReason, nextAction: failRes.nextAction },
|
||||||
|
checkpoint,
|
||||||
|
{ currentPhaseIdx: checkpoint.currentPhase, phaseResult: failRes }
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (checkpoint.currentPhase === this.phases.length) {
|
|
||||||
await this.saveCheckpoint(checkpoint);
|
checkpoint.inProgress = false;
|
||||||
await appendPhaseUpdate({
|
await this.saveCheckpoint(checkpoint);
|
||||||
eventType: 'workflow_completed',
|
await appendPhaseUpdate({
|
||||||
phase: null,
|
eventType: 'workflow_completed',
|
||||||
summary: `Workflow completed successfully`,
|
phase: null,
|
||||||
details: { totalPhases: this.phases.length },
|
summary: 'Workflow completed successfully',
|
||||||
});
|
details: { totalPhases: stages.length },
|
||||||
await this.writeStatus({ overallStatus: 'completed', nextAction: 'done' }, { currentPhaseIdx: null });
|
});
|
||||||
}
|
await this.writeStatus(stages, { overallStatus: 'completed', nextAction: 'done' }, checkpoint, { currentPhaseIdx: null });
|
||||||
|
}
|
||||||
|
|
||||||
|
public async start(stages: WorkflowStage[], context: WorkflowContext): Promise<void> {
|
||||||
|
await this.executeStages(stages, context, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async resume(stages: WorkflowStage[], context: WorkflowContext): Promise<void> {
|
||||||
|
await this.executeStages(stages, context, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Backward-compatible API used by existing tests.
|
||||||
|
public async run(): Promise<void> {
|
||||||
|
const phases = this.phases ?? [];
|
||||||
|
const input = this.input as TInput;
|
||||||
|
const stages = this.toStagesFromLegacyPhases(phases, input);
|
||||||
|
const context: WorkflowContext = {
|
||||||
|
runId: `legacy-${new Date().toISOString()}`,
|
||||||
|
startedAt: new Date().toISOString(),
|
||||||
|
metadata: { source: 'SequentialOrchestrator.run' },
|
||||||
|
};
|
||||||
|
await this.resume(stages, context);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue