fix(run): address gadfly review of the checkpoint PR
executus CI / test (pull_request) Successful in 45s

Real findings from the consensus review (44 raw; heavy devstral noise):

- finalizeCheckpoint is now fired from the top-of-Run defer, so it runs on
  EVERY exit: a panic, an early build-error return (before the run loop), AND
  normal completion. Previously an early return on a recovered run left its
  durable record unfinalized → boot recovery would retry it forever on a
  persistent build error. (opus + glm)
- Removed the dead ActivePhase field from run.RunCheckpointState +
  run.ResumeState (and the battery RunCheckpoint) — phase recovery is
  boundary-granular (skip completed phases; the interrupted phase re-runs from
  its start), so ActivePhase was never written nor read. Docs across
  ports/checkpoint/phases now state this plainly (5-model consensus that the
  field + docs over-promised mid-phase resume).
- CheckpointerFactory.Begin error is now logged (WARN) before degrading to
  non-durable, per the documented contract (was silently swallowed). (4 models)
- finalizeCheckpoint logs Complete/Fail errors (was silent).
- Resume phase-skip now keys off a SEPARATE resumeSkip set, not the live
  outputs map — a fresh run with two same-named phases no longer skips the
  second (the outputs map fills as phases run). (opus:max) + regression test.
- Removed the dead checkpoint.factory.now field (never set). (opus + glm)
- Fixed the stale phaseDeps doc (the step observer moved out of sharedOpts to
  per-path). Hoisted the resume guard to a local; dropped the wasted acc
  allocation on the resume path; documented that Save throttling is the
  Checkpointer's responsibility and the accumulated transcript is pre-compaction
  (host size-caps it).

Note (carried from the PR): classifyCheckpointOutcome keys shutdown on
run.ErrShutdown; mort stamps its own runengine.ErrShutdown — the mort wiring PR
aliases them so errors.Is matches.

New test: duplicate phase names both run on a fresh run. Full ./... green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-29 16:34:42 -04:00
parent 899059a791
commit 38d656ec71
7 changed files with 98 additions and 39 deletions
+1 -2
View File
@@ -35,10 +35,9 @@ type RunCheckpointMeta struct {
// RunCheckpoint is one persisted snapshot of a run's resumable progress. // RunCheckpoint is one persisted snapshot of a run's resumable progress.
type RunCheckpoint struct { type RunCheckpoint struct {
Meta RunCheckpointMeta Meta RunCheckpointMeta
Messages []llm.Message // conversation so far (single-loop or active phase) Messages []llm.Message // conversation so far (single-loop runs)
Iteration int // completed agent-loop iterations Iteration int // completed agent-loop iterations
CompletedPhases []run.PhaseOutput // finished phases, in order (multi-phase agents) CompletedPhases []run.PhaseOutput // finished phases, in order (multi-phase agents)
ActivePhase string // current phase name (multi-phase agents); "" otherwise
UpdatedAt time.Time UpdatedAt time.Time
} }
+1 -3
View File
@@ -58,7 +58,6 @@ func (h *handle) Save(ctx context.Context, st run.RunCheckpointState) error {
Messages: st.Messages, Messages: st.Messages,
Iteration: st.Iteration, Iteration: st.Iteration,
CompletedPhases: st.CompletedPhases, CompletedPhases: st.CompletedPhases,
ActivePhase: st.ActivePhase,
UpdatedAt: now, UpdatedAt: now,
}); err != nil { }); err != nil {
return err return err
@@ -92,7 +91,6 @@ func (noop) Fail(context.Context, error) error { return nil }
type factory struct { type factory struct {
store CheckpointStore store CheckpointStore
throttle time.Duration throttle time.Duration
now func() time.Time
} }
var _ run.CheckpointerFactory = (*factory)(nil) var _ run.CheckpointerFactory = (*factory)(nil)
@@ -119,5 +117,5 @@ func (f *factory) Begin(_ context.Context, info run.RunInfo) (run.Checkpointer,
ModelTier: info.ModelTier, ModelTier: info.ModelTier,
ParentRunID: info.ParentRunID, ParentRunID: info.ParentRunID,
} }
return New(f.store, meta, f.throttle, f.now), nil return New(f.store, meta, f.throttle, nil /* now defaults to time.Now */), nil
} }
+15 -6
View File
@@ -3,6 +3,7 @@ package run
import ( import (
"context" "context"
"errors" "errors"
"log/slog"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm" "gitea.stevedudenhoeffer.com/steve/majordomo/llm"
) )
@@ -17,12 +18,14 @@ import (
// ResumeState carries a recovered run's prior progress into Run so the run // ResumeState carries a recovered run's prior progress into Run so the run
// continues instead of restarting. The host's recovery path sets it via // continues instead of restarting. The host's recovery path sets it via
// WithResumeState; the executor reads it (single-loop seeds the saved transcript // WithResumeState; the executor reads it:
// as history; multi-phase skips completed phases and seeds the active phase). // - single-loop: History seeds the saved transcript (the run continues).
// - multi-phase: CompletedPhases are skipped; the interrupted phase re-runs
// from its start (boundary-granular — there is no mid-phase transcript
// resume, so History is unused for multi-phase runs).
type ResumeState struct { type ResumeState struct {
History []llm.Message // single-loop transcript OR active-phase transcript History []llm.Message // single-loop transcript (unused for multi-phase)
CompletedPhases []PhaseOutput // multi-phase: outputs of finished phases, in order CompletedPhases []PhaseOutput // multi-phase: outputs of finished phases, in order
ActivePhase string // multi-phase: the phase that was in flight
} }
type resumeStateKey struct{} type resumeStateKey struct{}
@@ -79,15 +82,21 @@ func classifyCheckpointOutcome(runErr, cause error) checkpointOutcome {
// finalizeCheckpoint applies the outcome to the per-run checkpointer (nil-safe). // finalizeCheckpoint applies the outcome to the per-run checkpointer (nil-safe).
// Runs on a detached context so a cancelled run still records its terminal state. // Runs on a detached context so a cancelled run still records its terminal state.
// Complete/Fail errors are best-effort but logged (a stale record would only
// cause a wasteful boot-recovery retry, not data loss).
func finalizeCheckpoint(ctx context.Context, cp Checkpointer, runErr error, cause error) { func finalizeCheckpoint(ctx context.Context, cp Checkpointer, runErr error, cause error) {
if cp == nil { if cp == nil {
return return
} }
switch classifyCheckpointOutcome(runErr, cause) { switch classifyCheckpointOutcome(runErr, cause) {
case checkpointComplete: case checkpointComplete:
_ = cp.Complete(detach(ctx)) if err := cp.Complete(detach(ctx)); err != nil {
slog.Warn("run: checkpoint Complete failed", "error", err)
}
case checkpointFail: case checkpointFail:
_ = cp.Fail(detach(ctx), runErr) if err := cp.Fail(detach(ctx), runErr); err != nil {
slog.Warn("run: checkpoint Fail failed", "error", err)
}
case checkpointLeaveRunning: case checkpointLeaveRunning:
// Interrupted by shutdown: leave the record for boot recovery. // Interrupted by shutdown: leave the record for boot recovery.
} }
+41 -12
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"log/slog"
"time" "time"
"gitea.stevedudenhoeffer.com/steve/majordomo/agent" "gitea.stevedudenhoeffer.com/steve/majordomo/agent"
@@ -113,13 +114,26 @@ type Result struct {
func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocation, input string) (res Result) { func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocation, input string) (res Result) {
started := time.Now() started := time.Now()
res = Result{RunID: inv.RunID} res = Result{RunID: inv.RunID}
// ckpt is the per-run durable checkpointer (resolved below; nil = non-durable).
// checkpointCause yields the run context's cancellation cause once the run
// context exists; nil before then (an early build-error return).
var ckpt Checkpointer
var checkpointCause func() error
// Enforce the no-panic contract: a panic anywhere in the run (incl. a host // Enforce the no-panic contract: a panic anywhere in the run (incl. a host
// Critic/Audit/Palette callback on the main goroutine) becomes Result.Err // Critic/Audit/Palette callback on the main goroutine) becomes Result.Err
// rather than unwinding into the caller. // rather than unwinding into the caller. This defer ALSO finalizes the
// checkpoint on EVERY exit path — panic, an early build-error return (before
// the run loop), or normal completion — so a recovered run's durable record is
// never left dangling (which would loop boot-recovery on a persistent error).
defer func() { defer func() {
if r := recover(); r != nil { if r := recover(); r != nil {
res.Err = fmt.Errorf("run.Executor: recovered panic: %v", r) res.Err = fmt.Errorf("run.Executor: recovered panic: %v", r)
} }
var cause error
if checkpointCause != nil {
cause = checkpointCause()
}
finalizeCheckpoint(ctx, ckpt, res.Err, cause)
}() }()
tier := ra.ModelTier tier := ra.ModelTier
@@ -188,9 +202,15 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
// Checkpointer via the factory (which decides durability — nil = non-durable). // Checkpointer via the factory (which decides durability — nil = non-durable).
// nil-safe throughout. // nil-safe throughout.
resume := resumeStateFromContext(ctx) resume := resumeStateFromContext(ctx)
ckpt := existingCheckpointerFromContext(ctx) ckpt = existingCheckpointerFromContext(ctx)
if ckpt == nil && e.cfg.Ports.Checkpointer != nil { if ckpt == nil && e.cfg.Ports.Checkpointer != nil {
if c, cerr := e.cfg.Ports.Checkpointer.Begin(ctx, info); cerr == nil { c, cerr := e.cfg.Ports.Checkpointer.Begin(ctx, info)
if cerr != nil {
// Degrade to non-durable (the documented contract) but log it — a
// failing checkpoint store must not fail the run, yet shouldn't be silent.
slog.Warn("run: checkpointer Begin failed; running non-durable",
"run_id", inv.RunID, "error", cerr)
} else {
ckpt = c ckpt = c
} }
} }
@@ -262,6 +282,9 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
defer cancelCause(nil) defer cancelCause(nil)
runCtx, mergeCancel := MergeCancellation(runCtx, ctx) runCtx, mergeCancel := MergeCancellation(runCtx, ctx)
defer mergeCancel() defer mergeCancel()
// The finalize defer (top of Run) now has a run context to read the
// cancellation cause from (shutdown vs critic-kill vs deadline vs cancel).
checkpointCause = func() error { return context.Cause(runCtx) }
// Critic (optional): monitors the run for a stall, can nudge/extend/kill via // Critic (optional): monitors the run for a stall, can nudge/extend/kill via
// its host Escalator. Its hard deadline is bound to runCtx (cancel on pass). // its host Escalator. Its hard deadline is bound to runCtx (cancel on pass).
@@ -339,6 +362,8 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
// the critic's nudges before each step. // the critic's nudges before each step.
steer := func() []llm.Message { return append(mailbox.drain(), critic.drainSteer()...) } steer := func() []llm.Message { return append(mailbox.drain(), critic.drainSteer()...) }
resuming := resume != nil && len(resume.History) > 0
var runRes *agent.Result var runRes *agent.Result
var runErr error var runErr error
if len(ra.Phases) == 0 { if len(ra.Phases) == 0 {
@@ -347,14 +372,19 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
// healthy-but-long run's budget mid-flight; falls back to maxIter). // healthy-but-long run's budget mid-flight; falls back to maxIter).
// //
// Checkpointing: wrap the step observer to accumulate the running transcript // Checkpointing: wrap the step observer to accumulate the running transcript
// and Save it each step (the host throttles). A recovered run seeds the saved // and Save it each step. Save is called every step; THROTTLING is the
// transcript as history and continues with no new input. acc starts from the // Checkpointer's responsibility (the battery + mort's durable-job adapter
// resume history (or the opening user message) and grows as steps complete. // both throttle + size-cap), so the kernel doesn't gate the hot path. The
// accumulated transcript is the pre-compaction one (the observer sees raw
// step responses, not the loop's compacted history) — a host that caps size
// bounds it. A recovered run seeds the saved transcript and continues.
obs := stepObserver obs := stepObserver
if ckpt != nil { if ckpt != nil {
acc := []llm.Message{multimodalUserMessage(input, inv.Images)} var acc []llm.Message
if resume != nil && len(resume.History) > 0 { if resuming {
acc = append([]llm.Message(nil), resume.History...) acc = append([]llm.Message(nil), resume.History...)
} else {
acc = []llm.Message{multimodalUserMessage(input, inv.Images)}
} }
obs = func(s agent.Step) { obs = func(s agent.Step) {
stepObserver(s) stepObserver(s)
@@ -373,7 +403,7 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
agent.WithStepObserver(obs), agent.WithStepObserver(obs),
}, sharedOpts...) }, sharedOpts...)
ag := agent.New(model, e.systemPrompt(ra), opts...) ag := agent.New(model, e.systemPrompt(ra), opts...)
if resume != nil && len(resume.History) > 0 { if resuming {
// Resume: seed the saved transcript and continue (no new input — the // Resume: seed the saved transcript and continue (no new input — the
// completed tool calls in the transcript are NOT re-run). // completed tool calls in the transcript are NOT re-run).
runRes, runErr = ag.Run(runCtx, "", agent.WithSteer(steer), agent.WithHistory(resume.History)) runRes, runErr = ag.Run(runCtx, "", agent.WithSteer(steer), agent.WithHistory(resume.History))
@@ -399,9 +429,8 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
}, input, inv.Images) }, input, inv.Images)
} }
// Finalize durable recovery: clear the checkpoint on success/terminal failure, // Durable-recovery finalize (Complete/Fail/leave-running) happens in the
// or leave it for boot recovery when the run was interrupted by shutdown. // top-of-Run defer so it covers panics + early build-error returns too.
finalizeCheckpoint(ctx, ckpt, runErr, context.Cause(runCtx))
status := statusFor(runCtx, runErr) status := statusFor(runCtx, runErr)
if runRes != nil { if runRes != nil {
+15 -11
View File
@@ -53,9 +53,10 @@ import (
// phaseDeps carries the per-run state the phase runner shares with Run: the base // phaseDeps carries the per-run state the phase runner shares with Run: the base
// model, the full decorated toolbox (filtered per phase), the base step cap, the // model, the full decorated toolbox (filtered per phase), the base step cap, the
// shared agent options (tool-error limits + step observer + compactor), the // shared agent options (tool-error limits + compactor — the step observer is
// shared step observer (also fed by IsRunFunc bare calls), the critic/session // added per phase, NOT in sharedOpts, so checkpointing can vary per path), the
// steer, and the audit recorder (phase events). // shared step observer (wired into each phase's loop AND invoked for IsRunFunc
// bare calls), the critic/session steer, and the audit recorder (phase events).
type phaseDeps struct { type phaseDeps struct {
baseModel llm.Model baseModel llm.Model
baseToolbox *llm.Toolbox baseToolbox *llm.Toolbox
@@ -85,12 +86,18 @@ func (e *Executor) runPhases(runCtx context.Context, ra RunnableAgent, deps phas
var lastOutput string var lastOutput string
var totalUsage llm.Usage var totalUsage llm.Usage
// Resume: pre-populate from the saved checkpoint so already-finished phases are // resumeSkip is the set of phases already finished on a RECOVERED run — kept
// skipped. The interrupted (active) phase is NOT pre-populated, so it re-runs // SEPARATE from the live `outputs` map (which fills as phases run this time) so
// from its start (boundary-granular recovery). // the skip guard only skips RESUME-completed phases, never a fresh run's own
// phases. (Reusing `outputs` would make a second phase with a duplicate name
// skip itself.) Pre-populate outputs + completed so a resumed run threads the
// saved outputs into later phases. The interrupted (active) phase is NOT
// pre-populated, so it re-runs from its start (boundary-granular recovery).
resumeSkip := map[string]bool{}
if deps.resume != nil { if deps.resume != nil {
for _, pc := range deps.resume.CompletedPhases { for _, pc := range deps.resume.CompletedPhases {
outputs[pc.Name] = pc.Output outputs[pc.Name] = pc.Output
resumeSkip[pc.Name] = true
completed = append(completed, pc) completed = append(completed, pc)
lastOutput = pc.Output lastOutput = pc.Output
} }
@@ -109,10 +116,8 @@ func (e *Executor) runPhases(runCtx context.Context, ra RunnableAgent, deps phas
} }
for i, phase := range ra.Phases { for i, phase := range ra.Phases {
// Skip phases already completed on a resumed run (key presence, not output // Skip phases already completed on a resumed run.
// emptiness — a legitimately-empty phase output still counts as done). if resumeSkip[phase.Name] {
if _, done := outputs[phase.Name]; done {
lastOutput = outputs[phase.Name]
continue continue
} }
// A killed/timed-out/cancelled run must not start its next phase. // A killed/timed-out/cancelled run must not start its next phase.
@@ -183,7 +188,6 @@ func (e *Executor) runPhases(runCtx context.Context, ra RunnableAgent, deps phas
if deps.checkpointer != nil { if deps.checkpointer != nil {
_ = deps.checkpointer.Save(runCtx, RunCheckpointState{ _ = deps.checkpointer.Save(runCtx, RunCheckpointState{
CompletedPhases: append([]PhaseOutput(nil), completed...), CompletedPhases: append([]PhaseOutput(nil), completed...),
ActivePhase: "",
}) })
} }
} }
+20
View File
@@ -130,6 +130,26 @@ func TestPhases_OptionalDoesNotSwallowCancellation(t *testing.T) {
} }
} }
// TestPhases_DuplicateNamesBothRun: a fresh (non-resume) run with two phases
// sharing a name must run BOTH — the resume-skip guard keys off a separate
// resume set, not the live outputs map (which fills as phases run), so a phase
// never skips a same-named sibling on a fresh run.
func TestPhases_DuplicateNamesBothRun(t *testing.T) {
models, fp := phaseProvider(t, fake.Reply("first"), fake.Reply("second"))
ex := New(Config{Registry: tool.NewRegistry(), Models: models})
ra := RunnableAgent{
Name: "p", ModelTier: "test-model",
Phases: []Phase{{Name: "x", SystemPrompt: "P1"}, {Name: "x", SystemPrompt: "P2"}},
}
res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r"}, "Q")
if res.Err != nil {
t.Fatalf("run error: %v", res.Err)
}
if n := len(fp.Calls()); n != 2 {
t.Fatalf("both same-named phases must run on a fresh run; got %d model calls", n)
}
}
// TestPhases_HardErrorAborts: a NON-optional phase that hits a hard error (not a // TestPhases_HardErrorAborts: a NON-optional phase that hits a hard error (not a
// budget/step exhaustion) aborts the pipeline; later phases do not run. // budget/step exhaustion) aborts the pipeline; later phases do not run.
func TestPhases_HardErrorAborts(t *testing.T) { func TestPhases_HardErrorAborts(t *testing.T) {
+5 -5
View File
@@ -199,15 +199,15 @@ type Checkpointer interface {
// RunCheckpointState is the resumable snapshot a Checkpointer persists. // RunCheckpointState is the resumable snapshot a Checkpointer persists.
type RunCheckpointState struct { type RunCheckpointState struct {
// Messages is the running transcript (single-loop run) OR the active phase's // Messages is the running transcript of a SINGLE-LOOP run (grows each step;
// transcript (multi-phase run). May be nil. // resumed via WithHistory). nil for multi-phase runs — phase recovery is
// boundary-granular (see CompletedPhases), not mid-phase transcript.
Messages []llm.Message Messages []llm.Message
Iteration int Iteration int
// CompletedPhases is set only for multi-phase runs: the outputs of phases // CompletedPhases is set only for multi-phase runs: the outputs of phases
// already finished, in phase order. nil for single-loop runs. // already finished, in phase order, so a resumed run skips them and re-runs
// the interrupted phase from its start. nil for single-loop runs.
CompletedPhases []PhaseOutput CompletedPhases []PhaseOutput
// ActivePhase is the name of the in-progress phase (multi-phase only).
ActivePhase string
} }
// PhaseOutput is one completed pipeline phase's name and output text, recorded in // PhaseOutput is one completed pipeline phase's name and output text, recorded in