fix(run): address gadfly review of the checkpoint PR
executus CI / test (pull_request) Successful in 45s
executus CI / test (pull_request) Successful in 45s
Real findings from the consensus review (44 raw; heavy devstral noise): - finalizeCheckpoint is now fired from the top-of-Run defer, so it runs on EVERY exit: a panic, an early build-error return (before the run loop), AND normal completion. Previously an early return on a recovered run left its durable record unfinalized → boot recovery would retry it forever on a persistent build error. (opus + glm) - Removed the dead ActivePhase field from run.RunCheckpointState + run.ResumeState (and the battery RunCheckpoint) — phase recovery is boundary-granular (skip completed phases; the interrupted phase re-runs from its start), so ActivePhase was never written nor read. Docs across ports/checkpoint/phases now state this plainly (5-model consensus that the field + docs over-promised mid-phase resume). - CheckpointerFactory.Begin error is now logged (WARN) before degrading to non-durable, per the documented contract (was silently swallowed). (4 models) - finalizeCheckpoint logs Complete/Fail errors (was silent). - Resume phase-skip now keys off a SEPARATE resumeSkip set, not the live outputs map — a fresh run with two same-named phases no longer skips the second (the outputs map fills as phases run). (opus:max) + regression test. - Removed the dead checkpoint.factory.now field (never set). (opus + glm) - Fixed the stale phaseDeps doc (the step observer moved out of sharedOpts to per-path). Hoisted the resume guard to a local; dropped the wasted acc allocation on the resume path; documented that Save throttling is the Checkpointer's responsibility and the accumulated transcript is pre-compaction (host size-caps it). Note (carried from the PR): classifyCheckpointOutcome keys shutdown on run.ErrShutdown; mort stamps its own runengine.ErrShutdown — the mort wiring PR aliases them so errors.Is matches. New test: duplicate phase names both run on a fresh run. Full ./... green. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+41
-12
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
|
||||
@@ -113,13 +114,26 @@ type Result struct {
|
||||
func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocation, input string) (res Result) {
|
||||
started := time.Now()
|
||||
res = Result{RunID: inv.RunID}
|
||||
// ckpt is the per-run durable checkpointer (resolved below; nil = non-durable).
|
||||
// checkpointCause yields the run context's cancellation cause once the run
|
||||
// context exists; nil before then (an early build-error return).
|
||||
var ckpt Checkpointer
|
||||
var checkpointCause func() error
|
||||
// Enforce the no-panic contract: a panic anywhere in the run (incl. a host
|
||||
// Critic/Audit/Palette callback on the main goroutine) becomes Result.Err
|
||||
// rather than unwinding into the caller.
|
||||
// rather than unwinding into the caller. This defer ALSO finalizes the
|
||||
// checkpoint on EVERY exit path — panic, an early build-error return (before
|
||||
// the run loop), or normal completion — so a recovered run's durable record is
|
||||
// never left dangling (which would loop boot-recovery on a persistent error).
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
res.Err = fmt.Errorf("run.Executor: recovered panic: %v", r)
|
||||
}
|
||||
var cause error
|
||||
if checkpointCause != nil {
|
||||
cause = checkpointCause()
|
||||
}
|
||||
finalizeCheckpoint(ctx, ckpt, res.Err, cause)
|
||||
}()
|
||||
|
||||
tier := ra.ModelTier
|
||||
@@ -188,9 +202,15 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
// Checkpointer via the factory (which decides durability — nil = non-durable).
|
||||
// nil-safe throughout.
|
||||
resume := resumeStateFromContext(ctx)
|
||||
ckpt := existingCheckpointerFromContext(ctx)
|
||||
ckpt = existingCheckpointerFromContext(ctx)
|
||||
if ckpt == nil && e.cfg.Ports.Checkpointer != nil {
|
||||
if c, cerr := e.cfg.Ports.Checkpointer.Begin(ctx, info); cerr == nil {
|
||||
c, cerr := e.cfg.Ports.Checkpointer.Begin(ctx, info)
|
||||
if cerr != nil {
|
||||
// Degrade to non-durable (the documented contract) but log it — a
|
||||
// failing checkpoint store must not fail the run, yet shouldn't be silent.
|
||||
slog.Warn("run: checkpointer Begin failed; running non-durable",
|
||||
"run_id", inv.RunID, "error", cerr)
|
||||
} else {
|
||||
ckpt = c
|
||||
}
|
||||
}
|
||||
@@ -262,6 +282,9 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
defer cancelCause(nil)
|
||||
runCtx, mergeCancel := MergeCancellation(runCtx, ctx)
|
||||
defer mergeCancel()
|
||||
// The finalize defer (top of Run) now has a run context to read the
|
||||
// cancellation cause from (shutdown vs critic-kill vs deadline vs cancel).
|
||||
checkpointCause = func() error { return context.Cause(runCtx) }
|
||||
|
||||
// Critic (optional): monitors the run for a stall, can nudge/extend/kill via
|
||||
// its host Escalator. Its hard deadline is bound to runCtx (cancel on pass).
|
||||
@@ -339,6 +362,8 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
// the critic's nudges before each step.
|
||||
steer := func() []llm.Message { return append(mailbox.drain(), critic.drainSteer()...) }
|
||||
|
||||
resuming := resume != nil && len(resume.History) > 0
|
||||
|
||||
var runRes *agent.Result
|
||||
var runErr error
|
||||
if len(ra.Phases) == 0 {
|
||||
@@ -347,14 +372,19 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
// healthy-but-long run's budget mid-flight; falls back to maxIter).
|
||||
//
|
||||
// Checkpointing: wrap the step observer to accumulate the running transcript
|
||||
// and Save it each step (the host throttles). A recovered run seeds the saved
|
||||
// transcript as history and continues with no new input. acc starts from the
|
||||
// resume history (or the opening user message) and grows as steps complete.
|
||||
// and Save it each step. Save is called every step; THROTTLING is the
|
||||
// Checkpointer's responsibility (the battery + mort's durable-job adapter
|
||||
// both throttle + size-cap), so the kernel doesn't gate the hot path. The
|
||||
// accumulated transcript is the pre-compaction one (the observer sees raw
|
||||
// step responses, not the loop's compacted history) — a host that caps size
|
||||
// bounds it. A recovered run seeds the saved transcript and continues.
|
||||
obs := stepObserver
|
||||
if ckpt != nil {
|
||||
acc := []llm.Message{multimodalUserMessage(input, inv.Images)}
|
||||
if resume != nil && len(resume.History) > 0 {
|
||||
var acc []llm.Message
|
||||
if resuming {
|
||||
acc = append([]llm.Message(nil), resume.History...)
|
||||
} else {
|
||||
acc = []llm.Message{multimodalUserMessage(input, inv.Images)}
|
||||
}
|
||||
obs = func(s agent.Step) {
|
||||
stepObserver(s)
|
||||
@@ -373,7 +403,7 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
agent.WithStepObserver(obs),
|
||||
}, sharedOpts...)
|
||||
ag := agent.New(model, e.systemPrompt(ra), opts...)
|
||||
if resume != nil && len(resume.History) > 0 {
|
||||
if resuming {
|
||||
// Resume: seed the saved transcript and continue (no new input — the
|
||||
// completed tool calls in the transcript are NOT re-run).
|
||||
runRes, runErr = ag.Run(runCtx, "", agent.WithSteer(steer), agent.WithHistory(resume.History))
|
||||
@@ -399,9 +429,8 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
}, input, inv.Images)
|
||||
}
|
||||
|
||||
// Finalize durable recovery: clear the checkpoint on success/terminal failure,
|
||||
// or leave it for boot recovery when the run was interrupted by shutdown.
|
||||
finalizeCheckpoint(ctx, ckpt, runErr, context.Cause(runCtx))
|
||||
// Durable-recovery finalize (Complete/Fail/leave-running) happens in the
|
||||
// top-of-Run defer so it covers panics + early build-error returns too.
|
||||
|
||||
status := statusFor(runCtx, runErr)
|
||||
if runRes != nil {
|
||||
|
||||
Reference in New Issue
Block a user