fix(run): address gadfly review of the checkpoint PR
executus CI / test (pull_request) Successful in 45s

Real findings from the consensus review (44 raw; heavy devstral noise):

- finalizeCheckpoint is now fired from the top-of-Run defer, so it runs on
  EVERY exit: a panic, an early build-error return (before the run loop), AND
  normal completion. Previously an early return on a recovered run left its
  durable record unfinalized → boot recovery would retry it forever on a
  persistent build error. (opus + glm)
- Removed the dead ActivePhase field from run.RunCheckpointState +
  run.ResumeState (and the battery RunCheckpoint) — phase recovery is
  boundary-granular (skip completed phases; the interrupted phase re-runs from
  its start), so ActivePhase was never written nor read. Docs across
  ports/checkpoint/phases now state this plainly (5-model consensus that the
  field + docs over-promised mid-phase resume).
- CheckpointerFactory.Begin error is now logged (WARN) before degrading to
  non-durable, per the documented contract (was silently swallowed). (4 models)
- finalizeCheckpoint logs Complete/Fail errors (was silent).
- Resume phase-skip now keys off a SEPARATE resumeSkip set, not the live
  outputs map — a fresh run with two same-named phases no longer skips the
  second (the outputs map fills as phases run). (opus:max) + regression test.
- Removed the dead checkpoint.factory.now field (never set). (opus + glm)
- Fixed the stale phaseDeps doc (the step observer moved out of sharedOpts to
  per-path). Hoisted the resume guard to a local; dropped the wasted acc
  allocation on the resume path; documented that Save throttling is the
  Checkpointer's responsibility and the accumulated transcript is pre-compaction
  (host size-caps it).

Note (carried from the PR): classifyCheckpointOutcome keys shutdown on
run.ErrShutdown; mort stamps its own runengine.ErrShutdown — the mort wiring PR
aliases them so errors.Is matches.

New test: duplicate phase names both run on a fresh run. Full ./... green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-29 16:34:42 -04:00
parent 899059a791
commit 38d656ec71
7 changed files with 98 additions and 39 deletions
+41 -12
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"log/slog"
"time"
"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
@@ -113,13 +114,26 @@ type Result struct {
func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocation, input string) (res Result) {
started := time.Now()
res = Result{RunID: inv.RunID}
// ckpt is the per-run durable checkpointer (resolved below; nil = non-durable).
// checkpointCause yields the run context's cancellation cause once the run
// context exists; nil before then (an early build-error return).
var ckpt Checkpointer
var checkpointCause func() error
// Enforce the no-panic contract: a panic anywhere in the run (incl. a host
// Critic/Audit/Palette callback on the main goroutine) becomes Result.Err
// rather than unwinding into the caller.
// rather than unwinding into the caller. This defer ALSO finalizes the
// checkpoint on EVERY exit path — panic, an early build-error return (before
// the run loop), or normal completion — so a recovered run's durable record is
// never left dangling (which would loop boot-recovery on a persistent error).
defer func() {
if r := recover(); r != nil {
res.Err = fmt.Errorf("run.Executor: recovered panic: %v", r)
}
var cause error
if checkpointCause != nil {
cause = checkpointCause()
}
finalizeCheckpoint(ctx, ckpt, res.Err, cause)
}()
tier := ra.ModelTier
@@ -188,9 +202,15 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
// Checkpointer via the factory (which decides durability — nil = non-durable).
// nil-safe throughout.
resume := resumeStateFromContext(ctx)
ckpt := existingCheckpointerFromContext(ctx)
ckpt = existingCheckpointerFromContext(ctx)
if ckpt == nil && e.cfg.Ports.Checkpointer != nil {
if c, cerr := e.cfg.Ports.Checkpointer.Begin(ctx, info); cerr == nil {
c, cerr := e.cfg.Ports.Checkpointer.Begin(ctx, info)
if cerr != nil {
// Degrade to non-durable (the documented contract) but log it — a
// failing checkpoint store must not fail the run, yet shouldn't be silent.
slog.Warn("run: checkpointer Begin failed; running non-durable",
"run_id", inv.RunID, "error", cerr)
} else {
ckpt = c
}
}
@@ -262,6 +282,9 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
defer cancelCause(nil)
runCtx, mergeCancel := MergeCancellation(runCtx, ctx)
defer mergeCancel()
// The finalize defer (top of Run) now has a run context to read the
// cancellation cause from (shutdown vs critic-kill vs deadline vs cancel).
checkpointCause = func() error { return context.Cause(runCtx) }
// Critic (optional): monitors the run for a stall, can nudge/extend/kill via
// its host Escalator. Its hard deadline is bound to runCtx (cancel on pass).
@@ -339,6 +362,8 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
// the critic's nudges before each step.
steer := func() []llm.Message { return append(mailbox.drain(), critic.drainSteer()...) }
resuming := resume != nil && len(resume.History) > 0
var runRes *agent.Result
var runErr error
if len(ra.Phases) == 0 {
@@ -347,14 +372,19 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
// healthy-but-long run's budget mid-flight; falls back to maxIter).
//
// Checkpointing: wrap the step observer to accumulate the running transcript
// and Save it each step (the host throttles). A recovered run seeds the saved
// transcript as history and continues with no new input. acc starts from the
// resume history (or the opening user message) and grows as steps complete.
// and Save it each step. Save is called every step; THROTTLING is the
// Checkpointer's responsibility (the battery + mort's durable-job adapter
// both throttle + size-cap), so the kernel doesn't gate the hot path. The
// accumulated transcript is the pre-compaction one (the observer sees raw
// step responses, not the loop's compacted history) — a host that caps size
// bounds it. A recovered run seeds the saved transcript and continues.
obs := stepObserver
if ckpt != nil {
acc := []llm.Message{multimodalUserMessage(input, inv.Images)}
if resume != nil && len(resume.History) > 0 {
var acc []llm.Message
if resuming {
acc = append([]llm.Message(nil), resume.History...)
} else {
acc = []llm.Message{multimodalUserMessage(input, inv.Images)}
}
obs = func(s agent.Step) {
stepObserver(s)
@@ -373,7 +403,7 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
agent.WithStepObserver(obs),
}, sharedOpts...)
ag := agent.New(model, e.systemPrompt(ra), opts...)
if resume != nil && len(resume.History) > 0 {
if resuming {
// Resume: seed the saved transcript and continue (no new input — the
// completed tool calls in the transcript are NOT re-run).
runRes, runErr = ag.Run(runCtx, "", agent.WithSteer(steer), agent.WithHistory(resume.History))
@@ -399,9 +429,8 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
}, input, inv.Images)
}
// Finalize durable recovery: clear the checkpoint on success/terminal failure,
// or leave it for boot recovery when the run was interrupted by shutdown.
finalizeCheckpoint(ctx, ckpt, runErr, context.Cause(runCtx))
// Durable-recovery finalize (Complete/Fail/leave-running) happens in the
// top-of-Run defer so it covers panics + early build-error returns too.
status := statusFor(runCtx, runErr)
if runRes != nil {