import { promises as fs } from 'fs'; import path from 'path'; import { logPhaseProgress } from './PhaseProgressLogger'; import { WorkflowStatusManager, WorkflowStatus } from './WorkflowStatusManager'; import { appendPhaseUpdate } from './PhaseUpdateQueue'; export type OrchestratorPhase = { name: string; run: (input: TInput) => Promise; retry?: number; backoffMs?: number; nextAction?: string; }; export type PhaseAttemptResult = { phase: string; status: 'success' | 'failure'; attempt: number; timestamp: string; error?: string; failureReason?: string; nextAction: string; }; export type Checkpoint = { currentPhase: number; phaseResults: PhaseAttemptResult[]; inProgress?: boolean; }; const DEFAULT_CHECKPOINT_FILE = path.join(process.cwd(), 'data/orchestrator-checkpoint.json'); const DEFAULT_STATUS_FILE = path.join(process.cwd(), 'status/workflow-status.json'); export class SequentialOrchestrator { private phases: OrchestratorPhase[]; private checkpointPath: string; private input: TInput; private maxAttemptsPerPhasePerRun: number; private statusManager: WorkflowStatusManager; constructor(options: { phases: OrchestratorPhase[]; checkpointPath?: string; input: TInput; maxAttemptsPerPhasePerRun?: number; statusFilePath?: string; }) { this.phases = options.phases; this.checkpointPath = options.checkpointPath || DEFAULT_CHECKPOINT_FILE; this.input = options.input; this.maxAttemptsPerPhasePerRun = options.maxAttemptsPerPhasePerRun ?? Infinity; this.statusManager = new WorkflowStatusManager(options.statusFilePath || DEFAULT_STATUS_FILE); } private async writeStatus( state: Partial, extra: { currentPhaseIdx?: number; phaseResult?: PhaseAttemptResult } = {} ) { const checkpoint = await this.loadCheckpoint(); let completedPhases: string[] = []; if (checkpoint) { completedPhases = this.phases .slice(0, checkpoint.currentPhase) .map(p => p.name) .filter(phaseName => checkpoint.phaseResults.some(r => r.phase === phaseName && r.status === 'success')); } const currentP = typeof extra.currentPhaseIdx === 'number' && this.phases[extra.currentPhaseIdx]?.name ? this.phases[extra.currentPhaseIdx].name : null; let overallStatus = state.overallStatus || 'idle'; if (extra.phaseResult && extra.phaseResult.status === 'failure') { overallStatus = 'failed'; } else if (checkpoint && checkpoint.currentPhase === this.phases.length) { overallStatus = 'completed'; } const lastFailureReason = (state.lastFailureReason === undefined && extra.phaseResult?.failureReason) ? extra.phaseResult?.failureReason || null : (state.lastFailureReason !== undefined ? state.lastFailureReason : null); const workflowStatus: WorkflowStatus = { currentPhase: currentP, overallStatus, lastUpdated: new Date().toISOString(), lastFailureReason, nextAction: state.nextAction || extra.phaseResult?.nextAction || '', completedPhases, }; await this.statusManager.update(workflowStatus); } private async loadCheckpoint(): Promise { try { const txt = await fs.readFile(this.checkpointPath, 'utf8'); return JSON.parse(txt) as Checkpoint; } catch { return null; } } private async saveCheckpoint(checkpoint: Checkpoint) { await fs.mkdir(path.dirname(this.checkpointPath), { recursive: true }); await fs.writeFile(this.checkpointPath, JSON.stringify(checkpoint, null, 2), 'utf8'); } public async run(): Promise { let checkpoint = await this.loadCheckpoint(); if (!checkpoint) { checkpoint = { currentPhase: 0, phaseResults: [] }; await this.saveCheckpoint(checkpoint); await this.writeStatus({ overallStatus: 'idle' }, { currentPhaseIdx: 0 }); } while (checkpoint.currentPhase < this.phases.length) { const phase = this.phases[checkpoint.currentPhase]; let success = false; let attempt = 1; const maxRetries = (typeof phase.retry === 'number') ? phase.retry : 1; const maxAttemptsThisRun = Number.isFinite(this.maxAttemptsPerPhasePerRun) ? Math.min(maxRetries, this.maxAttemptsPerPhasePerRun) : maxRetries; await appendPhaseUpdate({ eventType: 'phase_started', phase: phase.name, summary: `Phase '${phase.name}' started`, details: { attempt }, }); for (; attempt <= maxAttemptsThisRun && !success; attempt++) { try { await phase.run(this.input); const res: PhaseAttemptResult = { phase: phase.name, status: 'success', attempt, timestamp: new Date().toISOString(), nextAction: 'proceed', }; checkpoint.phaseResults.push(res); checkpoint.currentPhase += 1; await this.saveCheckpoint(checkpoint); await logPhaseProgress({ phase: res.phase, status: res.status, attempt: res.attempt, timestamp: res.timestamp, nextAction: res.nextAction, }); await appendPhaseUpdate({ eventType: 'phase_succeeded', phase: phase.name, summary: `Phase '${phase.name}' succeeded`, details: { attempt }, }); success = true; await this.writeStatus({ overallStatus: 'running', nextAction: phase.nextAction || '' }, { currentPhaseIdx: checkpoint.currentPhase, phaseResult: res }); } catch (err: any) { const isLastAttempt = attempt === maxRetries; // Always log every failed attempt const failRes: PhaseAttemptResult = { phase: phase.name, status: 'failure', attempt, timestamp: new Date().toISOString(), error: err && err.message ? err.message : String(err), failureReason: err && err.message ? err.message : String(err), nextAction: !isLastAttempt ? 'retry' : 'manual intervention', }; checkpoint.phaseResults.push(failRes); await logPhaseProgress({ phase: failRes.phase, status: failRes.status, attempt: failRes.attempt, timestamp: failRes.timestamp, failureReason: failRes.failureReason, nextAction: failRes.nextAction, }); if (isLastAttempt) { await appendPhaseUpdate({ eventType: 'phase_failed', phase: phase.name, summary: `Phase '${phase.name}' failed`, details: { attempt, error: err && err.message ? err.message : String(err) }, }); await this.saveCheckpoint(checkpoint); await this.writeStatus({ overallStatus: 'failed', lastFailureReason: failRes.failureReason, nextAction: failRes.nextAction }, { currentPhaseIdx: checkpoint.currentPhase, phaseResult: failRes }); return; } } } } if (checkpoint.currentPhase === this.phases.length) { await this.saveCheckpoint(checkpoint); await appendPhaseUpdate({ eventType: 'workflow_completed', phase: null, summary: `Workflow completed successfully`, details: { totalPhases: this.phases.length }, }); await this.writeStatus({ overallStatus: 'completed', nextAction: 'done' }, { currentPhaseIdx: null }); } } }