Merge pull request 'feat(run): critic owns the deadline — MaxRuntime becomes the soft trigger' (#21) from feat/critic-owns-deadline into main

All low-severity follow-ups on the critic-deadline change:
- hardCap uses max(CriticAbsoluteMax, maxRuntime) instead of a nested if (723193a7).
- Drop the now-dead 90s soft-trigger fallback + its bare literal: the sole caller
  passes the resolved MaxRuntime (>0), and Run's unsupervised-run failsafe bounds
  even an impossible 0 (8d377051, 2f86bf58).
- Decouple the kernel doc from a named downstream convar ("a 6h host convar")
  (730c67fc).

Graded false-positive: agent.go BackstopMultiplier validation (handled in the host;
not in this diff), the 24h default "magic number" (matches every withFallbacks
default), and the defer-in-conditional pattern (idiomatic). Kept: the thorough
two-tier comment (this logic regressed once) and the rare-path nested timer.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Jo75sqmeVPgFUWZQBn179X

checkpoint/checkpoint.go

@@ -4,9 +4,9 @@
 // run.Ports.Checkpointer.
 //
 // Mort backs CheckpointStore with its durable-job table; Memory() is the
-// zero-dependency default; contrib/store can add a SQLite one. NOTE: the
+// zero-dependency default; contrib/store can add a SQLite one. The executor calls
-// executor's call into run.Ports.Checkpointer is a P2 follow-up — this battery
+// run.Ports.Checkpointer (a CheckpointerFactory) during the run loop; NewFactory
-// provides the seam + impls ahead of that wiring.
+// wires this battery into that seam.
 package checkpoint
 
 import (
@@ -14,6 +14,8 @@ import (
 	"time"
 
 	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
+
+	"gitea.stevedudenhoeffer.com/steve/executus/run"
 )
 
 // RunCheckpointMeta is the run attribution needed to resume a run from scratch
@@ -32,11 +34,11 @@ type RunCheckpointMeta struct {
 
 // RunCheckpoint is one persisted snapshot of a run's resumable progress.
 type RunCheckpoint struct {
-	Meta        RunCheckpointMeta
+	Meta            RunCheckpointMeta
-	Messages    []llm.Message // conversation so far
+	Messages        []llm.Message     // conversation so far (single-loop runs)
-	Iteration   int           // completed agent-loop iterations
+	Iteration       int               // completed agent-loop iterations
-	ActivePhase string        // current phase name (multi-phase agents); "" otherwise
+	CompletedPhases []run.PhaseOutput // finished phases, in order (multi-phase agents)
-	UpdatedAt   time.Time
+	UpdatedAt       time.Time
 }
 
 // CheckpointStore persists run checkpoints keyed by run id. A live checkpoint

checkpoint/handle.go

@@ -54,10 +54,11 @@ func (h *handle) Save(ctx context.Context, st run.RunCheckpointState) error {
 	// caller believes was saved. (A run drives one Save goroutine, so the brief
 	// unguarded window here can't double-write.)
 	if err := h.store.Save(ctx, RunCheckpoint{
-		Meta:      h.meta,
+		Meta:            h.meta,
-		Messages:  st.Messages,
+		Messages:        st.Messages,
-		Iteration: st.Iteration,
+		Iteration:       st.Iteration,
-		UpdatedAt: now,
+		CompletedPhases: st.CompletedPhases,
+		UpdatedAt:       now,
 	}); err != nil {
 		return err
 	}
@@ -81,3 +82,40 @@ var _ run.Checkpointer = noop{}
 func (noop) Save(context.Context, run.RunCheckpointState) error { return nil }
 func (noop) Complete(context.Context) error                     { return nil }
 func (noop) Fail(context.Context, error) error                  { return nil }
+
+// factory is a run.CheckpointerFactory that mints a per-run handle over store,
+// deriving the per-run meta from the kernel's RunInfo. It is the battery's glue
+// for the Ports.Checkpointer (factory) seam: every run becomes durable (the
+// store persists snapshots; a host wanting lazy/short-run skipping uses its own
+// factory, as mort does over its durable-job table).
+type factory struct {
+	store    CheckpointStore
+	throttle time.Duration
+}
+
+var _ run.CheckpointerFactory = (*factory)(nil)
+
+// NewFactory returns a run.CheckpointerFactory backed by store: each run gets a
+// per-run Checkpointer (throttled to at most once per throttle). A nil store
+// yields factory.Begin returning a no-op Checkpointer.
+func NewFactory(store CheckpointStore, throttle time.Duration) run.CheckpointerFactory {
+	return &factory{store: store, throttle: throttle}
+}
+
+// Begin mints the per-run Checkpointer. The prompt is read from
+// info.Inputs["prompt"] when present so a recovered run can re-dispatch.
+func (f *factory) Begin(_ context.Context, info run.RunInfo) (run.Checkpointer, error) {
+	prompt, _ := info.Inputs["prompt"].(string)
+	meta := RunCheckpointMeta{
+		RunID:       info.RunID,
+		AgentID:     info.SubjectID,
+		AgentName:   info.Name,
+		CallerID:    info.CallerID,
+		ChannelID:   info.ChannelID,
+		GuildID:     info.GuildID,
+		Prompt:      prompt,
+		ModelTier:   info.ModelTier,
+		ParentRunID: info.ParentRunID,
+	}
+	return New(f.store, meta, f.throttle, nil /* now defaults to time.Now */), nil
+}

run/agent.go

@@ -55,15 +55,27 @@ type RunnableAgent struct {
 }
 
 // Phase is one step of a multi-step run: its own system prompt, model tier,
-// iteration cap, and tool subset. Optional phases may be skipped by the
+// iteration cap, and tool subset. Phase prompts are Go text/template strings
-// pipeline when their precondition isn't met.
+// expanded against {{.Query}} (the original input) and {{.<PhaseName>}} (a
+// prior phase's output) before the phase runs, so a phase can consume earlier
+// work. The final phase's output is the run's output.
 type Phase struct {
 	Name          string
 	SystemPrompt  string
 	ModelTier     string
 	MaxIterations int
 	Tools         []string
-	Optional      bool
+	// Optional swallows a phase's error and substitutes FallbackMessage (or a
+	// generated note) as its output, so a non-critical phase failing does not
+	// abort the pipeline.
+	Optional bool
+	// FallbackMessage is the substitute output when an Optional phase fails.
+	// Empty → a generated "(phase %q encountered an error…)" note.
+	FallbackMessage string
+	// IsRunFunc marks a phase as a single bare LLM call (no tool loop, no tools
+	// array) — a deterministic transform step (plan/synthesize) rather than an
+	// agentic loop. Its Tools/MaxIterations are ignored.
+	IsRunFunc bool
 }
 
 // CriticConfig configures the optional run-critic. Enabled gates whether a

run/checkpoint.go

@@ -0,0 +1,103 @@
+package run
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+
+	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
+)
+
+// Durable-recovery plumbing for the executor. The Checkpointer port (set via
+// Ports.Checkpointer, a CheckpointerFactory) persists a run's resumable progress
+// during the loop; on boot a host re-dispatches an interrupted run through the
+// executor with a ResumeState (the saved transcript / completed phases) so it
+// CONTINUES rather than restarting, reusing the SAME durable record via an
+// existing Checkpointer. Both are carried into Run via the context (mirrors
+// mort's agentexec.WithResumeState / WithExistingCheckpointer).
+
+// ResumeState carries a recovered run's prior progress into Run so the run
+// continues instead of restarting. The host's recovery path sets it via
+// WithResumeState; the executor reads it:
+//   - single-loop: History seeds the saved transcript (the run continues).
+//   - multi-phase: CompletedPhases are skipped; the interrupted phase re-runs
+//     from its start (boundary-granular — there is no mid-phase transcript
+//     resume, so History is unused for multi-phase runs).
+type ResumeState struct {
+	History         []llm.Message // single-loop transcript (unused for multi-phase)
+	CompletedPhases []PhaseOutput // multi-phase: outputs of finished phases, in order
+}
+
+type resumeStateKey struct{}
+
+// WithResumeState carries a recovered run's prior progress into Run.
+func WithResumeState(ctx context.Context, rs *ResumeState) context.Context {
+	return context.WithValue(ctx, resumeStateKey{}, rs)
+}
+
+func resumeStateFromContext(ctx context.Context) *ResumeState {
+	rs, _ := ctx.Value(resumeStateKey{}).(*ResumeState)
+	return rs
+}
+
+type existingCheckpointerKey struct{}
+
+// WithExistingCheckpointer carries a pre-existing Checkpointer into Run so a
+// recovery re-run reuses the SAME durable record (the executor uses it instead of
+// calling Ports.Checkpointer.Begin).
+func WithExistingCheckpointer(ctx context.Context, cp Checkpointer) context.Context {
+	return context.WithValue(ctx, existingCheckpointerKey{}, cp)
+}
+
+func existingCheckpointerFromContext(ctx context.Context) Checkpointer {
+	cp, _ := ctx.Value(existingCheckpointerKey{}).(Checkpointer)
+	return cp
+}
+
+// checkpointOutcome is the finalize decision for a durable run.
+type checkpointOutcome int
+
+const (
+	checkpointComplete checkpointOutcome = iota
+	checkpointLeaveRunning
+	checkpointFail
+)
+
+// classifyCheckpointOutcome maps (run error, cancellation cause) to the durable
+// finalize action: success clears the checkpoint (Complete); a shutdown-caused
+// cancellation leaves the record so boot recovery picks it up (neither
+// Complete nor Fail); anything else (model error, tool loop, the run's own
+// deadline, a critic kill, a caller cancel) is terminal (Fail). Mirrors mort's
+// agentexec.classifyCheckpointOutcome.
+func classifyCheckpointOutcome(runErr, cause error) checkpointOutcome {
+	switch {
+	case runErr == nil:
+		return checkpointComplete
+	case errors.Is(cause, ErrShutdown):
+		return checkpointLeaveRunning
+	default:
+		return checkpointFail
+	}
+}
+
+// finalizeCheckpoint applies the outcome to the per-run checkpointer (nil-safe).
+// Runs on a detached context so a cancelled run still records its terminal state.
+// Complete/Fail errors are best-effort but logged (a stale record would only
+// cause a wasteful boot-recovery retry, not data loss).
+func finalizeCheckpoint(ctx context.Context, cp Checkpointer, runErr error, cause error) {
+	if cp == nil {
+		return
+	}
+	switch classifyCheckpointOutcome(runErr, cause) {
+	case checkpointComplete:
+		if err := cp.Complete(detach(ctx)); err != nil {
+			slog.Warn("run: checkpoint Complete failed", "error", err)
+		}
+	case checkpointFail:
+		if err := cp.Fail(detach(ctx), runErr); err != nil {
+			slog.Warn("run: checkpoint Fail failed", "error", err)
+		}
+	case checkpointLeaveRunning:
+		// Interrupted by shutdown: leave the record for boot recovery.
+	}
+}

run/checkpoint_test.go

@@ -0,0 +1,200 @@
+package run
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
+	"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
+
+	"gitea.stevedudenhoeffer.com/steve/executus/tool"
+)
+
+// fakeCheckpointer records every Save state + whether Complete/Fail fired.
+type fakeCheckpointer struct {
+	saves     []RunCheckpointState
+	completed bool
+	failed    bool
+	failErr   error
+}
+
+func (c *fakeCheckpointer) Save(_ context.Context, st RunCheckpointState) error {
+	c.saves = append(c.saves, st)
+	return nil
+}
+func (c *fakeCheckpointer) Complete(context.Context) error { c.completed = true; return nil }
+func (c *fakeCheckpointer) Fail(_ context.Context, err error) error {
+	c.failed = true
+	c.failErr = err
+	return nil
+}
+
+// fakeCheckpointFactory hands out one fakeCheckpointer and records the RunInfo.
+type fakeCheckpointFactory struct {
+	cp   *fakeCheckpointer
+	info RunInfo
+}
+
+func (f *fakeCheckpointFactory) Begin(_ context.Context, info RunInfo) (Checkpointer, error) {
+	f.info = info
+	return f.cp, nil
+}
+
+// TestClassifyCheckpointOutcome covers the finalize decision matrix.
+func TestClassifyCheckpointOutcome(t *testing.T) {
+	cases := []struct {
+		name  string
+		err   error
+		cause error
+		want  checkpointOutcome
+	}{
+		{"success", nil, nil, checkpointComplete},
+		{"shutdown", context.Canceled, ErrShutdown, checkpointLeaveRunning},
+		{"critic-kill", context.Canceled, ErrCriticKill, checkpointFail},
+		{"deadline", context.DeadlineExceeded, context.DeadlineExceeded, checkpointFail},
+		{"model-error", errors.New("boom"), nil, checkpointFail},
+		{"caller-cancel", context.Canceled, context.Canceled, checkpointFail},
+	}
+	for _, tc := range cases {
+		if got := classifyCheckpointOutcome(tc.err, tc.cause); got != tc.want {
+			t.Errorf("%s: classifyCheckpointOutcome = %v, want %v", tc.name, got, tc.want)
+		}
+	}
+}
+
+// TestCheckpoint_SingleLoopSaveAndComplete: a durable single-loop run gets a
+// per-run checkpointer (Begin), Saves its transcript each step, and Completes on
+// success (clearing the checkpoint). The RunInfo carries the resume meta.
+func TestCheckpoint_SingleLoopSaveAndComplete(t *testing.T) {
+	models, _ := phaseProvider(t, fake.Reply("done"))
+	cp := &fakeCheckpointer{}
+	f := &fakeCheckpointFactory{cp: cp}
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models, Ports: Ports{Checkpointer: f}})
+
+	res := ex.Run(context.Background(),
+		RunnableAgent{ID: "a1", Name: "boss", ModelTier: "test-model"},
+		tool.Invocation{RunID: "run-x", CallerID: "steve", ChannelID: "chan", GuildID: "g", SkillInputs: map[string]any{"prompt": "go"}},
+		"go")
+	if res.Err != nil {
+		t.Fatalf("run error: %v", res.Err)
+	}
+	if f.info.RunID != "run-x" || f.info.SubjectID != "a1" || f.info.ModelTier != "test-model" || f.info.GuildID != "g" {
+		t.Errorf("Begin RunInfo missing resume meta: %+v", f.info)
+	}
+	if len(cp.saves) == 0 {
+		t.Error("expected at least one checkpoint Save during the run")
+	} else if len(cp.saves[len(cp.saves)-1].Messages) == 0 {
+		t.Error("checkpoint Save should carry the running transcript")
+	}
+	if !cp.completed {
+		t.Error("a successful run must Complete (clear) its checkpoint")
+	}
+	if cp.failed {
+		t.Error("a successful run must NOT Fail its checkpoint")
+	}
+}
+
+// TestCheckpoint_TerminalErrorFails: a run that errors (not shutdown) Fails its
+// checkpoint (clears it — not a recovery candidate).
+func TestCheckpoint_TerminalErrorFails(t *testing.T) {
+	models, _ := phaseProvider(t, fake.Fail(errors.New("model down")))
+	cp := &fakeCheckpointer{}
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models, Ports: Ports{Checkpointer: &fakeCheckpointFactory{cp: cp}}})
+
+	res := ex.Run(context.Background(),
+		RunnableAgent{ID: "a1", ModelTier: "test-model"},
+		tool.Invocation{RunID: "r", CallerID: "c", SkillInputs: map[string]any{"prompt": "go"}}, "go")
+	if res.Err == nil {
+		t.Fatal("expected a run error")
+	}
+	if !cp.failed {
+		t.Error("a terminal (non-shutdown) error must Fail the checkpoint")
+	}
+	if cp.completed {
+		t.Error("a failed run must NOT Complete its checkpoint")
+	}
+}
+
+// TestCheckpoint_ResumeSeedsHistory: a run carrying a ResumeState seeds the saved
+// transcript as the model's opening messages (continues) instead of the input.
+func TestCheckpoint_ResumeSeedsHistory(t *testing.T) {
+	models, fp := phaseProvider(t, fake.Reply("continued"))
+	history := []llm.Message{llm.UserText("prior turn 1"), llm.AssistantText("prior answer 1")}
+	ctx := WithResumeState(context.Background(), &ResumeState{History: history})
+
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models})
+	res := ex.Run(ctx,
+		RunnableAgent{ID: "a1", ModelTier: "test-model"},
+		tool.Invocation{RunID: "r", CallerID: "c", SkillInputs: map[string]any{"prompt": "ignored-on-resume"}}, "ignored-on-resume")
+	if res.Err != nil {
+		t.Fatalf("run error: %v", res.Err)
+	}
+	got := fp.Calls()[0].Request.Messages
+	if len(got) != len(history) {
+		t.Fatalf("resume should seed the saved %d-message transcript, got %d messages", len(history), len(got))
+	}
+}
+
+// TestCheckpoint_PhaseBoundarySavesCompleted: a durable multi-phase run records
+// the completed phases at each boundary, growing the list, and Completes on
+// success.
+func TestCheckpoint_PhaseBoundarySavesCompleted(t *testing.T) {
+	models, _ := phaseProvider(t, fake.Reply("out-a"), fake.Reply("out-b"))
+	cp := &fakeCheckpointer{}
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models, Ports: Ports{Checkpointer: &fakeCheckpointFactory{cp: cp}}})
+
+	ra := RunnableAgent{
+		ID: "p", ModelTier: "test-model",
+		Phases: []Phase{{Name: "a", SystemPrompt: "A"}, {Name: "b", SystemPrompt: "B"}},
+	}
+	if res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r", CallerID: "c"}, "Q"); res.Err != nil {
+		t.Fatalf("run error: %v", res.Err)
+	}
+	// The final phase-boundary Save must list both completed phases.
+	var lastPhaseSave *RunCheckpointState
+	for i := range cp.saves {
+		if len(cp.saves[i].CompletedPhases) > 0 {
+			lastPhaseSave = &cp.saves[i]
+		}
+	}
+	if lastPhaseSave == nil || len(lastPhaseSave.CompletedPhases) != 2 {
+		t.Fatalf("expected a phase-boundary Save listing 2 completed phases; saves=%+v", cp.saves)
+	}
+	if !cp.completed {
+		t.Error("a successful phased run must Complete its checkpoint")
+	}
+}
+
+// TestCheckpoint_ResumeSkipsCompletedPhases: a resumed multi-phase run skips
+// phases already in ResumeState.CompletedPhases (only the remaining phase calls
+// the model) and threads their outputs into the remaining phase's template.
+func TestCheckpoint_ResumeSkipsCompletedPhases(t *testing.T) {
+	models, fp := phaseProvider(t, fake.Reply("out-b")) // ONLY phase b should call the model
+	ctx := WithResumeState(context.Background(), &ResumeState{
+		CompletedPhases: []PhaseOutput{{Name: "a", Output: "saved-a"}},
+	})
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models})
+
+	ra := RunnableAgent{
+		ID: "p", ModelTier: "test-model",
+		Phases: []Phase{
+			{Name: "a", SystemPrompt: "A"},
+			{Name: "b", SystemPrompt: "B saw {{.a}}"},
+		},
+	}
+	res := ex.Run(ctx, ra, tool.Invocation{RunID: "r", CallerID: "c"}, "Q")
+	if res.Err != nil {
+		t.Fatalf("run error: %v", res.Err)
+	}
+	if res.Output != "out-b" {
+		t.Fatalf("output = %q, want out-b", res.Output)
+	}
+	calls := fp.Calls()
+	if len(calls) != 1 {
+		t.Fatalf("only the un-completed phase b should call the model; got %d calls", len(calls))
+	}
+	if calls[0].Request.System != "B saw saved-a" {
+		t.Errorf("resumed phase b should see the completed phase a's saved output; system = %q", calls[0].Request.System)
+	}
+}

run/critic.go

@@ -22,6 +22,14 @@ type criticBinding struct {
 	h CriticHandle
 }
 
+// criticOwnsDeadline reports whether a critic is configured AND this run enables
+// it — the single predicate that decides the two-tier-timeout path. Used by BOTH
+// Run (to choose the generous runaway ceiling over the literal MaxRuntime cap) and
+// startCritic (the arm/no-op gate), so the two can never drift.
+func (e *Executor) criticOwnsDeadline(ra RunnableAgent) bool {
+	return e.cfg.Ports.Critic != nil && ra.Critic.Enabled
+}
+
 // startCritic begins critic monitoring for this run when one is configured and
 // the agent enables it. It launches a goroutine that cancels runCtx (via
 // cancelCause) the moment the critic's hard deadline passes — the critic may
@@ -31,16 +39,20 @@ type criticBinding struct {
 // "killed"); when the backstop simply expired, it is context.DeadlineExceeded (→
 // "timeout"). Returns (nil, no-op stop) when there is no critic. The caller MUST
 // defer the returned stop.
-func (e *Executor) startCritic(runCtx context.Context, cancelCause context.CancelCauseFunc, ra RunnableAgent, info RunInfo) (*criticBinding, func()) {
+//
+// softTrigger is the run's resolved MaxRuntime: for a critic-owned run MaxRuntime
+// is the soft wake (mort's two-tier semantics — the critic first reviews once the
+// run exceeds its nominal budget, and its backstop = softTrigger × multiplier).
+// The caller (Run) always passes the resolved MaxRuntime, which withFallbacks
+// guarantees is > 0, so no fallback is needed here. (A non-positive soft would make
+// the host Monitor return no handle, and Run's unsupervised-run failsafe then bounds
+// the run at MaxRuntime — so even that impossible case stays bounded.)
+func (e *Executor) startCritic(runCtx context.Context, cancelCause context.CancelCauseFunc, ra RunnableAgent, info RunInfo, softTrigger time.Duration) (*criticBinding, func()) {
 	noop := func() {}
-	if e.cfg.Ports.Critic == nil || !ra.Critic.Enabled {
+	if !e.criticOwnsDeadline(ra) {
 		return nil, noop
 	}
-	soft := e.cfg.Defaults.CriticSoftTimeout
+	h := e.cfg.Ports.Critic.Monitor(runCtx, info, softTrigger)
-	if soft <= 0 {
-		soft = 90 * time.Second // defensive: withFallbacks normally guarantees >0
-	}
-	h := e.cfg.Ports.Critic.Monitor(runCtx, info, soft)
 	if h == nil {
 		return nil, noop
 	}

run/critic_deadline_test.go

@@ -0,0 +1,153 @@
+package run_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
+	"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
+
+	"gitea.stevedudenhoeffer.com/steve/executus/run"
+	"gitea.stevedudenhoeffer.com/steve/executus/tool"
+)
+
+// slowToolInvocation builds an Invocation whose session factory adds a "slow"
+// tool that sleeps for d (respecting ctx). The model script calls it once, then
+// answers — so the run's wall-clock is dominated by d, letting a test set a tiny
+// MaxRuntime and observe whether MaxRuntime hard-cancels the run.
+func slowToolInvocation(runID string, d time.Duration) tool.Invocation {
+	slow := llm.DefineTool("slow", "sleeps for a while",
+		func(ctx context.Context, _ struct{}) (any, error) {
+			select {
+			case <-time.After(d):
+				return "ok", nil
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			}
+		})
+	return tool.Invocation{
+		RunID: runID,
+		SessionToolFactory: func(_ tool.AgentSession) tool.SessionTools {
+			return tool.SessionTools{Tools: []llm.Tool{slow}}
+		},
+	}
+}
+
+func slowModel() llm.Model {
+	fp := fake.New("fake")
+	fp.Enqueue("m",
+		fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c1", Name: "slow", Arguments: []byte(`{}`)}}}),
+		fake.Reply("done"),
+	)
+	m, _ := fp.Model("m")
+	return m
+}
+
+// TestNoCritic_MaxRuntimeIsHardCap: the legacy contract is preserved — without a
+// critic, MaxRuntime is a literal WithTimeout that kills a run whose work outlasts
+// it. The slow tool (200ms) outlasts MaxRuntime (20ms), so runCtx cancels mid-tool
+// and the run ends in error (timeout).
+func TestNoCritic_MaxRuntimeIsHardCap(t *testing.T) {
+	m := slowModel()
+	ex := run.New(run.Config{
+		Registry: tool.NewRegistry(),
+		Models:   func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
+	})
+	res := ex.Run(context.Background(),
+		run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 5, MaxRuntime: 20 * time.Millisecond},
+		slowToolInvocation("r", 200*time.Millisecond), "go")
+	if res.Err == nil {
+		t.Fatalf("non-critic run should hard-timeout at MaxRuntime; got output=%q err=nil", res.Output)
+	}
+}
+
+// TestCriticOwnsDeadline_SurvivesPastMaxRuntime: the fix — when the critic owns the
+// deadline (Ports.Critic set + Critic.Enabled), MaxRuntime becomes the SOFT trigger
+// and is NOT a hard cap. The fake critic exposes no hard deadline (Deadline()==zero,
+// no kill), so the only hard ceiling is CriticAbsoluteMax (10s here). The slow tool
+// (200ms) outlasts the tiny MaxRuntime (20ms) but the run completes — proving the
+// old agentexec two-tier semantics are restored.
+func TestCriticOwnsDeadline_SurvivesPastMaxRuntime(t *testing.T) {
+	m := slowModel()
+	h := &fakeCriticHandle{} // Deadline()==zero → no hard deadline, no kill
+	ex := run.New(run.Config{
+		Registry: tool.NewRegistry(),
+		Models:   func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
+		Ports:    run.Ports{Critic: &fakeCritic{h: h}},
+		Defaults: run.Defaults{CriticAbsoluteMax: 10 * time.Second},
+	})
+	res := ex.Run(context.Background(),
+		run.RunnableAgent{Name: "watched", ModelTier: "m", MaxIterations: 5, MaxRuntime: 20 * time.Millisecond,
+			Critic: run.CriticConfig{Enabled: true}},
+		slowToolInvocation("r", 200*time.Millisecond), "go")
+	if res.Err != nil {
+		t.Fatalf("critic-owned run must survive past MaxRuntime (soft trigger); got err=%v", res.Err)
+	}
+	if res.Output != "done" {
+		t.Errorf("output = %q, want %q", res.Output, "done")
+	}
+}
+
+// capturingCritic records the soft trigger the executor passes to Monitor.
+type capturingCritic struct {
+	mu   sync.Mutex
+	soft time.Duration
+	h    run.CriticHandle
+}
+
+func (c *capturingCritic) Monitor(_ context.Context, _ run.RunInfo, soft time.Duration) run.CriticHandle {
+	c.mu.Lock()
+	c.soft = soft
+	c.mu.Unlock()
+	return c.h
+}
+
+// TestCriticSoftTriggerIsMaxRuntime: the soft trigger handed to the host critic is
+// the run's resolved MaxRuntime (mort's two-tier model — the critic first wakes once
+// the run exceeds its nominal budget), not some global/default value.
+func TestCriticSoftTriggerIsMaxRuntime(t *testing.T) {
+	fp := fake.New("fake")
+	fp.Enqueue("m", fake.Reply("done"))
+	m, _ := fp.Model("m")
+	cc := &capturingCritic{h: &fakeCriticHandle{}}
+	ex := run.New(run.Config{
+		Registry: tool.NewRegistry(),
+		Models:   func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
+		Ports:    run.Ports{Critic: cc},
+	})
+	const wantSoft = 7 * time.Minute
+	ex.Run(context.Background(),
+		run.RunnableAgent{Name: "x", ModelTier: "m", MaxRuntime: wantSoft, Critic: run.CriticConfig{Enabled: true}},
+		tool.Invocation{RunID: "r"}, "go")
+	cc.mu.Lock()
+	got := cc.soft
+	cc.mu.Unlock()
+	if got != wantSoft {
+		t.Errorf("soft trigger = %v, want the agent's MaxRuntime %v", got, wantSoft)
+	}
+}
+
+// TestCriticOwnsDeadline_NilHandleFallsBackToMaxRuntime: the agent enables the
+// critic but the host Monitor returns NO handle (nil) — there is no deadline-watch,
+// so the run is unsupervised. It must fall back to the nominal MaxRuntime hard cap
+// (the slow 200ms tool outlasts the 20ms MaxRuntime → the run errors), NOT run free
+// up to the generous CriticAbsoluteMax runaway ceiling.
+func TestCriticOwnsDeadline_NilHandleFallsBackToMaxRuntime(t *testing.T) {
+	m := slowModel()
+	cc := &capturingCritic{} // h is the nil interface → Monitor returns a nil handle
+	ex := run.New(run.Config{
+		Registry: tool.NewRegistry(),
+		Models:   func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
+		Ports:    run.Ports{Critic: cc},
+		Defaults: run.Defaults{CriticAbsoluteMax: time.Hour}, // generous ceiling; must NOT be what bounds the run
+	})
+	res := ex.Run(context.Background(),
+		run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 5, MaxRuntime: 20 * time.Millisecond,
+			Critic: run.CriticConfig{Enabled: true}},
+		slowToolInvocation("r", 200*time.Millisecond), "go")
+	if res.Err == nil {
+		t.Fatalf("critic-enabled run with a nil Monitor handle must fall back to the MaxRuntime hard cap; got output=%q err=nil", res.Output)
+	}
+}

run/critic_test.go

@@ -61,8 +61,8 @@ func TestCriticRaisesStepCeiling(t *testing.T) {
 		Registry: tool.NewRegistry(),
 		Models:   func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
 		Ports:    run.Ports{Critic: &fakeCritic{h: h}},
-		// large soft timeout so the deadline-watch never interferes in the test
+		// The fake handle's Deadline() is zero (no hard deadline), so the
-		Defaults: run.Defaults{CriticSoftTimeout: time.Hour},
+		// deadline-watch never interferes regardless of the soft trigger.
 	})
 	res := ex.Run(context.Background(),
 		run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 1, Critic: run.CriticConfig{Enabled: true}},

run/executor.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"strings"
+	"log/slog"
 	"time"
 
 	"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
@@ -29,7 +29,17 @@ type Defaults struct {
 	MaxConsecutiveToolErrors int           // loop guard; default 3
 	MaxSameToolCallRepeats   int           // retry-storm guard; default 3
 	CompactionThresholdRatio float64       // fraction of model context to compact at; default 0.7
-	CriticSoftTimeout        time.Duration // idle window before the critic wakes; default 90s
+	// CriticAbsoluteMax is the RUNAWAY ceiling for a critic-OWNED run (Ports.Critic
+	// set AND the agent enables it). For such a run MaxRuntime is the SOFT trigger,
+	// not a hard cap, and the critic's own extendable backstop is the normal
+	// deadline. This ceiling exists ONLY to stop a critic that never advances its
+	// deadline (a broken host handle) from running forever, so it is deliberately
+	// set FAR beyond any realistic backstop (default 24h): the host clamps its own
+	// backstop to a much smaller absolute max (e.g. a 6h host convar), so the ceiling
+	// never pre-empts a healthy supervised run. Keep it well above the host's
+	// absolute max. Never shorter than the run's MaxRuntime. Non-critic runs ignore
+	// it (they keep the literal MaxRuntime kill).
+	CriticAbsoluteMax time.Duration
 }
 
 func (d Defaults) withFallbacks() Defaults {
@@ -51,8 +61,8 @@ func (d Defaults) withFallbacks() Defaults {
 	if d.CompactionThresholdRatio <= 0 {
 		d.CompactionThresholdRatio = 0.7
 	}
-	if d.CriticSoftTimeout <= 0 {
+	if d.CriticAbsoluteMax <= 0 {
-		d.CriticSoftTimeout = 90 * time.Second
+		d.CriticAbsoluteMax = 24 * time.Hour
 	}
 	return d
 }
@@ -114,13 +124,26 @@ type Result struct {
 func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocation, input string) (res Result) {
 	started := time.Now()
 	res = Result{RunID: inv.RunID}
+	// ckpt is the per-run durable checkpointer (resolved below; nil = non-durable).
+	// checkpointCause yields the run context's cancellation cause once the run
+	// context exists; nil before then (an early build-error return).
+	var ckpt Checkpointer
+	var checkpointCause func() error
 	// Enforce the no-panic contract: a panic anywhere in the run (incl. a host
 	// Critic/Audit/Palette callback on the main goroutine) becomes Result.Err
-	// rather than unwinding into the caller.
+	// rather than unwinding into the caller. This defer ALSO finalizes the
+	// checkpoint on EVERY exit path — panic, an early build-error return (before
+	// the run loop), or normal completion — so a recovered run's durable record is
+	// never left dangling (which would loop boot-recovery on a persistent error).
 	defer func() {
 		if r := recover(); r != nil {
 			res.Err = fmt.Errorf("run.Executor: recovered panic: %v", r)
 		}
+		var cause error
+		if checkpointCause != nil {
+			cause = checkpointCause()
+		}
+		finalizeCheckpoint(ctx, ckpt, res.Err, cause)
 	}()
 
 	tier := ra.ModelTier
@@ -166,7 +189,9 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
 		Name:          ra.Name,
 		CallerID:      inv.CallerID,
 		ChannelID:     inv.ChannelID,
+		GuildID:       inv.GuildID,
 		ParentRunID:   inv.ParentRunID,
+		ModelTier:     tier,
 		Inputs:        inv.SkillInputs,
 		StartedAt:     started,
 		MaxIterations: maxIter,
@@ -181,6 +206,25 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
 		inv.RunState = stateAcc
 	}
 
+	// Durable recovery (optional): a recovered run carries a ResumeState (prior
+	// transcript / completed phases) + an existing Checkpointer in ctx so it
+	// continues on the SAME durable record; a fresh run mints a per-run
+	// Checkpointer via the factory (which decides durability — nil = non-durable).
+	// nil-safe throughout.
+	resume := resumeStateFromContext(ctx)
+	ckpt = existingCheckpointerFromContext(ctx)
+	if ckpt == nil && e.cfg.Ports.Checkpointer != nil {
+		c, cerr := e.cfg.Ports.Checkpointer.Begin(ctx, info)
+		if cerr != nil {
+			// Degrade to non-durable (the documented contract) but log it — a
+			// failing checkpoint store must not fail the run, yet shouldn't be silent.
+			slog.Warn("run: checkpointer Begin failed; running non-durable",
+				"run_id", inv.RunID, "error", cerr)
+		} else {
+			ckpt = c
+		}
+	}
+
 	// Steer mailbox: lets session tools (via inv.AttachImages) feed multimodal
 	// messages into the running conversation before its next step. Created BEFORE
 	// the toolbox build so any tool's handler captures the live AttachImages seam.
@@ -231,18 +275,40 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
 		postRun = st.PostRun
 	}
 
-	// Run context: bound by MaxRuntime, detached from the caller's deadline so a
+	// Run context: detached from the caller's deadline so a lane/queue wait doesn't
-	// lane/queue wait doesn't eat the run budget (mort's V10 lesson). Caller
+	// eat the run budget (mort's V10 lesson). Caller cancellation still propagates
-	// cancellation still propagates via MergeCancellation. Created BEFORE the
+	// via MergeCancellation. Created BEFORE the step observer so the observer
-	// step observer so the observer forwards the merged run context (not a
+	// forwards the merged run context (not a possibly-cancelled caller ctx) to
-	// possibly-cancelled caller ctx) to OnStep consumers.
+	// OnStep consumers.
-	// MaxRuntime stays a WithTimeout so its DeadlineExceeded propagates through the
+	//
-	// child chain (→ "timeout"), preserving the run's-own-timeout vs caller-cancel
+	// Two-tier timeout: who owns the hard deadline depends on the critic.
-	// distinction. A NESTED cause-carrying layer lets a critic kill surface as a
+	//   - NO critic (the default): MaxRuntime is a literal WithTimeout. Its
-	// distinct "killed" without disturbing that: only an ErrCriticKill cause is
+	//     DeadlineExceeded propagates through the child chain (→ "timeout"),
-	// consulted in statusFor; a generic run error or a caller cancel is classified
+	//     preserving the run's-own-timeout vs caller-cancel distinction.
-	// by the run error itself.
+	//   - critic OWNS the deadline (Ports.Critic set + ra.Critic.Enabled):
-	timeoutCtx, cancelTimeout := context.WithTimeout(context.WithoutCancel(ctx), maxRuntime)
+	//     MaxRuntime becomes the SOFT trigger (passed to startCritic), and the
+	//     critic's extendable backstop — watched in startCritic, which cancels via
+	//     cancelCause — is the real deadline. A slow-but-progressing run is given
+	//     room up to that backstop; only a stalled one is killed. The base context
+	//     gets a WithTimeout at CriticAbsoluteMax (default 24h) purely as a RUNAWAY
+	//     guard for a critic that never advances its deadline: it is set FAR beyond
+	//     any realistic backstop (the host clamps its own backstop to a much smaller
+	//     absolute max, e.g. a 6h host convar), so it does NOT pre-empt a healthy
+	//     supervised run. If the host critic fails to ARM (nil handle), the run is
+	//     unsupervised and we tighten the cap back down to MaxRuntime below.
+	// A NESTED cause-carrying layer (cancelCause) lets a critic kill surface as a
+	// distinct "killed": only an ErrCriticKill cause is consulted in statusFor; a
+	// generic run error, a backstop expiry, or a caller cancel is classified by the
+	// run error itself.
+	criticOwns := e.criticOwnsDeadline(ra)
+	hardCap := maxRuntime
+	if criticOwns {
+		// Runaway guard only — the critic's own (extendable) deadline-watch is the
+		// normal cap. max() keeps it from being shorter than the nominal budget if an
+		// operator sets MaxRuntime above the runaway ceiling (a degenerate config).
+		hardCap = max(e.cfg.Defaults.CriticAbsoluteMax, maxRuntime)
+	}
+	timeoutCtx, cancelTimeout := context.WithTimeout(context.WithoutCancel(ctx), hardCap)
 	defer cancelTimeout()
 	runCtx, cancelCause := context.WithCancelCause(timeoutCtx)
 	defer cancelCause(nil)
@@ -250,11 +316,29 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
 	defer mergeCancel()
 
 	// Critic (optional): monitors the run for a stall, can nudge/extend/kill via
-	// its host Escalator. Its hard deadline is bound to runCtx (cancel on pass).
+	// its host Escalator. When it owns the deadline, MaxRuntime is its soft trigger
-	// nil-safe: no-op when no critic is configured or the agent doesn't enable it.
+	// (so a slow-but-progressing run survives past it); its extendable backstop is
-	critic, stopCritic := e.startCritic(runCtx, cancelCause, ra, info)
+	// bound to runCtx (cancel on pass). nil-safe: no-op when no critic is configured
+	// or the agent doesn't enable it.
+	critic, stopCritic := e.startCritic(runCtx, cancelCause, ra, info, maxRuntime)
 	defer stopCritic()
 
+	// Unsupervised-run failsafe: the agent enabled the critic (so the base context
+	// got the generous runaway ceiling instead of MaxRuntime), but the host Monitor
+	// returned no handle — there is no deadline-watch. Without this the run would be
+	// bounded only by the 24h ceiling. Tighten it back to the nominal MaxRuntime so
+	// an unsupervised run can't hold its slot far past budget. mort's adapter always
+	// arms when the flag is set, so this is pure defence in depth.
+	if criticOwns && critic == nil {
+		var cancelUnsupervised context.CancelFunc
+		runCtx, cancelUnsupervised = context.WithTimeout(runCtx, maxRuntime)
+		defer cancelUnsupervised()
+	}
+	// The finalize defer (top of Run) now has a run context to read the
+	// cancellation cause from (shutdown vs critic-kill vs deadline vs cancel). Set
+	// AFTER the unsupervised-failsafe re-wrap so it reads the context the loop runs on.
+	checkpointCause = func() error { return context.Cause(runCtx) }
+
 	// Step instrumentation: accumulate Result.Steps + fire inv.OnStep, feed the
 	// audit recorder, and keep the live iteration counter fresh. majordomo's
 	// step observer hands us each completed iteration; we zip the model's tool
@@ -289,14 +373,12 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
 		}
 	}
 
-	opts := []agent.Option{
+	// Shared agent options used by BOTH the single-loop path and every phase: the
-		agent.WithToolbox(toolbox),
+	// tool-error guards and optional compaction. The toolbox, step ceiling, AND
-		// Step ceiling: a fixed WithMaxSteps(maxIter) normally, but when a critic is
+	// step observer are added per path (the observer is wrapped for checkpointing,
-		// active it owns a DYNAMIC ceiling (WithMaxStepsFunc) so it can raise a
+	// which differs single-loop vs per-phase).
-		// healthy-but-long run's budget mid-flight. Falls back to maxIter.
+	sharedOpts := []agent.Option{
-		critic.maxStepsOption(maxIter),
 		agent.WithToolErrorLimits(e.cfg.Defaults.MaxConsecutiveToolErrors, e.cfg.Defaults.MaxSameToolCallRepeats),
-		agent.WithStepObserver(stepObserver),
 	}
 	if e.cfg.Compactor != nil && e.cfg.ContextTokens != nil {
 		if threshold := e.compactionThreshold(tier); threshold > 0 {
@@ -313,11 +395,10 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
 					})
 				}
 			}
-			opts = append(opts, agent.WithCompactor(e.cfg.Compactor(threshold, onFire)))
+			sharedOpts = append(sharedOpts, agent.WithCompactor(e.cfg.Compactor(threshold, onFire)))
 		}
 	}
 
-	ag := agent.New(model, e.systemPrompt(ra), opts...)
 	// Stage non-image input attachments (audio/PDF/binary) into the host file
 	// store and fold an [ATTACHED FILES] descriptor into the prompt so the agent
 	// can reach them by file_id. No-op when Ports.InputFiles is nil or there are
@@ -327,7 +408,76 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
 	// One WithSteer drains BOTH the session mailbox (a tool's AttachImages) and
 	// the critic's nudges before each step.
 	steer := func() []llm.Message { return append(mailbox.drain(), critic.drainSteer()...) }
-	runRes, runErr := runAgent(runCtx, ag, input, inv.Images, agent.WithSteer(steer))
+
+	resuming := resume != nil && len(resume.History) > 0
+
+	var runRes *agent.Result
+	var runErr error
+	if len(ra.Phases) == 0 {
+		// Single-loop run: the agent's base prompt + full toolbox, with the
+		// critic's DYNAMIC step ceiling (WithMaxStepsFunc, so it can raise a
+		// healthy-but-long run's budget mid-flight; falls back to maxIter).
+		//
+		// Checkpointing: wrap the step observer to accumulate the running transcript
+		// and Save it each step. Save is called every step; THROTTLING is the
+		// Checkpointer's responsibility (the battery + mort's durable-job adapter
+		// both throttle + size-cap), so the kernel doesn't gate the hot path. The
+		// accumulated transcript is the pre-compaction one (the observer sees raw
+		// step responses, not the loop's compacted history) — a host that caps size
+		// bounds it. A recovered run seeds the saved transcript and continues.
+		obs := stepObserver
+		if ckpt != nil {
+			var acc []llm.Message
+			if resuming {
+				acc = append([]llm.Message(nil), resume.History...)
+			} else {
+				acc = []llm.Message{multimodalUserMessage(input, inv.Images)}
+			}
+			obs = func(s agent.Step) {
+				stepObserver(s)
+				if s.Response != nil {
+					acc = append(acc, s.Response.Message())
+				}
+				if len(s.Results) > 0 {
+					acc = append(acc, llm.ToolResultsMessage(s.Results...))
+				}
+				_ = ckpt.Save(runCtx, RunCheckpointState{Messages: acc, Iteration: s.Index + 1})
+			}
+		}
+		opts := append([]agent.Option{
+			agent.WithToolbox(toolbox),
+			critic.maxStepsOption(maxIter),
+			agent.WithStepObserver(obs),
+		}, sharedOpts...)
+		ag := agent.New(model, e.systemPrompt(ra), opts...)
+		if resuming {
+			// Resume: seed the saved transcript and continue (no new input — the
+			// completed tool calls in the transcript are NOT re-run).
+			runRes, runErr = ag.Run(runCtx, "", agent.WithSteer(steer), agent.WithHistory(resume.History))
+		} else {
+			runRes, runErr = runAgent(runCtx, ag, input, inv.Images, agent.WithSteer(steer))
+		}
+	} else {
+		// Multi-phase pipeline: each phase runs its own prompt/tier/tools/step-cap
+		// sequentially, threading outputs through {{.<PhaseName>}} templates. The
+		// shared step observer (audit/steps/critic) is wired per phase by the phase
+		// runner; checkpointing is phase-boundary granular (completed phases are
+		// recorded so a resumed run skips them).
+		runRes, runErr = e.runPhases(runCtx, ra, phaseDeps{
+			baseModel:    model,
+			baseToolbox:  toolbox,
+			baseMaxIter:  maxIter,
+			sharedOpts:   sharedOpts,
+			stepObserver: stepObserver,
+			steer:        steer,
+			rec:          rec,
+			checkpointer: ckpt,
+			resume:       resume,
+		}, input, inv.Images)
+	}
+
+	// Durable-recovery finalize (Complete/Fail/leave-running) happens in the
+	// top-of-Run defer so it covers panics + early build-error returns too.
 
 	status := statusFor(runCtx, runErr)
 	if runRes != nil {
@@ -403,13 +553,20 @@ func (e *Executor) finishAudit(ctx context.Context, rec RunRecorder, status stri
 }
 
 func (e *Executor) systemPrompt(ra RunnableAgent) string {
+	return e.systemPromptWithBody(ra.SystemPrompt)
+}
+
+// systemPromptWithBody composes the optional platform header with an arbitrary
+// body. The single-loop path passes ra.SystemPrompt; the phase runner passes a
+// phase's expanded instructions, so each phase keeps the platform header.
+func (e *Executor) systemPromptWithBody(body string) string {
 	if e.cfg.SystemHeader == "" {
-		return ra.SystemPrompt
+		return body
 	}
-	if ra.SystemPrompt == "" {
+	if body == "" {
 		return e.cfg.SystemHeader
 	}
-	return e.cfg.SystemHeader + "\n\n" + ra.SystemPrompt
+	return e.cfg.SystemHeader + "\n\n" + body
 }
 
 // compactionThreshold returns the token threshold for the tier's model context
@@ -460,15 +617,9 @@ func runAgent(ctx context.Context, ag *agent.Agent, input string, images []llm.I
 	if len(images) == 0 {
 		return ag.Run(ctx, input, opts...)
 	}
-	parts := make([]llm.Part, 0, len(images)+1)
-	if strings.TrimSpace(input) != "" {
-		parts = append(parts, llm.Text(input))
-	}
-	for _, img := range images {
-		parts = append(parts, img)
-	}
 	// Copy opts before appending so a caller-supplied backing array is never
-	// mutated/aliased (the variadic slice can have spare capacity).
+	// mutated/aliased (the variadic slice can have spare capacity). The multimodal
-	opts = append(opts[:len(opts):len(opts)], agent.WithHistory([]llm.Message{llm.UserParts(parts...)}))
+	// opening turn (text + image parts) is built by the shared helper.
+	opts = append(opts[:len(opts):len(opts)], agent.WithHistory([]llm.Message{multimodalUserMessage(input, images)}))
 	return ag.Run(ctx, "", opts...)
 }

run/phases.go

@@ -0,0 +1,398 @@
+package run
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"text/template"
+	"unicode/utf8"
+
+	"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
+	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
+)
+
+// The multi-step phase runner. A phased RunnableAgent (ra.Phases non-empty) runs
+// its phases in order; each phase is a fresh majordomo agent loop (or a single
+// bare LLM call for IsRunFunc phases) with its own template-expanded system
+// prompt, model tier, step cap, and tool subset. Phase outputs feed later phases
+// through {{.<PhaseName>}} template variables; {{.Query}} is the original input.
+// The final phase's output is the run's output.
+//
+// Ported from mort's agentexec pipeline so the executus kernel — which already
+// carries RunnableAgent.Phases as a DTO — actually EXECUTES them (it previously
+// ignored the slice and ran a single loop with the base prompt). It reuses the
+// shared run machinery built once in Run: the same stepObserver (so audit/steps/
+// critic-activity accumulate across every phase, including IsRunFunc bare calls),
+// the same critic steer, and the same compaction option.
+//
+// Semantics preserved from mort's pipeline:
+//   - phases run sequentially; ctx cancellation/deadline/critic-kill aborts the
+//     run (even mid-phase and even for an Optional phase).
+//   - IsRunFunc = one bare LLM call, no tools, no loop.
+//   - Optional phases swallow NON-context errors and substitute FallbackMessage.
+//   - a non-optional phase that merely exhausts its step/tool budget is NOT fatal:
+//     its partial transcript is salvaged and the pipeline continues — EXCEPT a
+//     final phase that salvaged nothing, which is a genuine empty-result failure.
+//   - per-phase ModelTier resolve failures fall back to the base model with a WARN.
+//
+// Deliberately NOT carried over (kernel is leaner than mort's legacy pipeline):
+// the legacy `submit` capture tool (the kernel relies on majordomo's
+// no-tool-call-is-final-answer termination, like its single-loop path), and the
+// critic's dynamic iteration ceiling (per-phase caps are fixed at phase start —
+// the run-level critic's steer + hard deadline still apply across phases).
+//
+// NOTE on phase names: {{.<PhaseName>}} resolves a map key, so a phase whose name
+// is not a Go-template identifier (hyphens, spaces, leading digit) cannot be
+// referenced as {{.my-phase}} — authors must use {{index . "my-phase"}}. A
+// template that fails to parse/execute is logged (WARN) and passed through
+// unchanged rather than silently dropped (see expandPhaseTemplate). Avoid naming
+// a phase "Query" — it shadows the original-input variable.
+
+// phaseDeps carries the per-run state the phase runner shares with Run: the base
+// model, the full decorated toolbox (filtered per phase), the base step cap, the
+// shared agent options (tool-error limits + compactor — the step observer is
+// added per phase, NOT in sharedOpts, so checkpointing can vary per path), the
+// shared step observer (wired into each phase's loop AND invoked for IsRunFunc
+// bare calls), the critic/session steer, and the audit recorder (phase events).
+type phaseDeps struct {
+	baseModel    llm.Model
+	baseToolbox  *llm.Toolbox
+	baseMaxIter  int
+	sharedOpts   []agent.Option
+	stepObserver func(agent.Step)
+	steer        func() []llm.Message
+	rec          RunRecorder
+	// checkpointer records phase-boundary progress (completed phases) for durable
+	// recovery; nil = non-durable. resume carries a recovered run's completed
+	// phases so they are skipped on re-run. Phase recovery is boundary-granular:
+	// the interrupted (active) phase re-runs from its start (its mid-phase
+	// transcript is NOT resumed — only the single-loop path resumes mid-loop).
+	checkpointer Checkpointer
+	resume       *ResumeState
+}
+
+// runPhases executes ra.Phases sequentially and returns a synthetic agent.Result
+// whose Output is the final phase's output, with Usage aggregated across phases
+// and Messages set to the last phase's transcript (for the PostRun hook). A hard
+// (non-optional, non-budget) phase failure — and any context cancellation/
+// deadline/critic-kill — returns the error.
+func (e *Executor) runPhases(runCtx context.Context, ra RunnableAgent, deps phaseDeps, query string, images []llm.ImagePart) (*agent.Result, error) {
+	outputs := make(map[string]string, len(ra.Phases))
+	var completed []PhaseOutput
+	var lastResult *agent.Result
+	var lastOutput string
+	var totalUsage llm.Usage
+
+	// resumeSkip is the set of phases already finished on a RECOVERED run — kept
+	// SEPARATE from the live `outputs` map (which fills as phases run this time) so
+	// the skip guard only skips RESUME-completed phases, never a fresh run's own
+	// phases. (Reusing `outputs` would make a second phase with a duplicate name
+	// skip itself.) Pre-populate outputs + completed so a resumed run threads the
+	// saved outputs into later phases. The interrupted (active) phase is NOT
+	// pre-populated, so it re-runs from its start (boundary-granular recovery).
+	resumeSkip := map[string]bool{}
+	if deps.resume != nil {
+		for _, pc := range deps.resume.CompletedPhases {
+			outputs[pc.Name] = pc.Output
+			resumeSkip[pc.Name] = true
+			completed = append(completed, pc)
+			lastOutput = pc.Output
+		}
+	}
+
+	// finish stamps the aggregated usage + final output onto the synthetic result.
+	finish := func(err error) (*agent.Result, error) {
+		if lastResult == nil {
+			lastResult = &agent.Result{}
+		}
+		lastResult.Usage = totalUsage
+		if err == nil {
+			lastResult.Output = lastOutput
+		}
+		return lastResult, err
+	}
+
+	for i, phase := range ra.Phases {
+		// Skip phases already completed on a resumed run.
+		if resumeSkip[phase.Name] {
+			continue
+		}
+		// A killed/timed-out/cancelled run must not start its next phase.
+		if err := runCtx.Err(); err != nil {
+			return finish(err)
+		}
+
+		instructions := expandPhaseTemplate(phase.SystemPrompt, query, outputs)
+		if deps.rec != nil {
+			deps.rec.LogEvent("phase_start", map[string]any{"phase": phase.Name})
+		}
+
+		output, res, err := e.runOnePhase(runCtx, ra, deps, phase, instructions, query, images)
+		if res != nil {
+			lastResult = res
+			totalUsage = addUsage(totalUsage, res.Usage)
+		}
+		if err != nil {
+			// A context cancellation / deadline / critic-kill is NEVER swallowed by
+			// the Optional or budget-salvage branches — the run genuinely ended and
+			// must surface as cancelled/timeout/killed (statusFor classifies it).
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return finish(err)
+			}
+			isLast := i == len(ra.Phases)-1
+			trimmed := strings.TrimSpace(output)
+			switch {
+			case phase.Optional:
+				output = phase.FallbackMessage
+				if output == "" {
+					output = fmt.Sprintf("(Phase %q encountered an error -- proceeding without its results)", phase.Name)
+				}
+				slog.Warn("run: optional pipeline phase failed",
+					"agent", ra.Name, "phase", phase.Name, "error", err)
+				if deps.rec != nil {
+					deps.rec.LogEvent("phase_failed_optional", map[string]any{"phase": phase.Name, "error": err.Error()})
+				}
+
+			case isPhaseBudgetExhaustion(err) && (!isLast || trimmed != ""):
+				// Soft stop: the phase ran out of its step/tool budget before
+				// composing a final answer. Not fatal — it did real work (runOnePhase
+				// salvaged its partial transcript into output), and aborting would
+				// discard every completed phase before it. Degrade and continue.
+				// (A FINAL phase that salvaged nothing falls through to the hard error
+				// below: there is no result to return.)
+				if trimmed == "" {
+					output = fmt.Sprintf("(Phase %q reached its step budget before producing a consolidated result; continuing with its partial findings.)", phase.Name)
+				} else {
+					output += fmt.Sprintf("\n\n(Note: phase %q reached its step budget before fully completing; the above is its partial output.)", phase.Name)
+				}
+				slog.Warn("run: pipeline phase exhausted its budget; salvaging partial output and continuing",
+					"agent", ra.Name, "phase", phase.Name, "last_phase", isLast, "error", err)
+				if deps.rec != nil {
+					deps.rec.LogEvent("phase_budget_exhausted", map[string]any{"phase": phase.Name, "error": err.Error(), "last_phase": isLast})
+				}
+
+			default:
+				return finish(fmt.Errorf("pipeline phase %q: %w", phase.Name, err))
+			}
+		}
+
+		outputs[phase.Name] = output
+		lastOutput = output
+		// Checkpoint the phase boundary: this phase is done, so a resumed run skips
+		// it and continues from the next. (Copy the slice — the checkpointer may
+		// hold/serialize it asynchronously.)
+		completed = append(completed, PhaseOutput{Name: phase.Name, Output: output})
+		if deps.checkpointer != nil {
+			_ = deps.checkpointer.Save(runCtx, RunCheckpointState{
+				CompletedPhases: append([]PhaseOutput(nil), completed...),
+			})
+		}
+	}
+
+	return finish(nil)
+}
+
+// runOnePhase runs a single phase: a bare LLM call for IsRunFunc phases, a fresh
+// agent loop otherwise. Returns the phase output, the loop result (nil for a
+// failed bare call), and any error. On a budget-exhaustion error the loop's
+// partial transcript is salvaged into the returned output.
+func (e *Executor) runOnePhase(runCtx context.Context, ra RunnableAgent, deps phaseDeps, phase Phase, instructions, query string, images []llm.ImagePart) (string, *agent.Result, error) {
+	phaseCtx, model := e.phaseModel(runCtx, deps, ra, phase)
+	// The phase's expanded instructions are the system prompt (with the platform
+	// header so tools keep their run ids); the original query is the user message.
+	system := e.systemPromptWithBody(instructions)
+
+	if phase.IsRunFunc {
+		// Bare LLM call: no tool loop, no tools array (some models 400 on an empty
+		// tools list). The response is fed through the SAME step observer as a loop
+		// step so the audit token tally, Result.Steps, AND the critic's activity
+		// clock all see it (a long synthesize phase must not look idle to the critic).
+		msgs := []llm.Message{multimodalUserMessage(query, images)}
+		resp, err := model.Generate(phaseCtx, llm.Request{System: system, Messages: msgs})
+		if err != nil {
+			return "", nil, fmt.Errorf("phase %q model call: %w", phase.Name, err)
+		}
+		if deps.stepObserver != nil {
+			deps.stepObserver(agent.Step{Index: 0, Response: resp})
+		}
+		return resp.Text(), &agent.Result{
+			Output:   resp.Text(),
+			Usage:    resp.Usage,
+			Messages: append(msgs, resp.Message()),
+		}, nil
+	}
+
+	toolbox := filterToolbox(deps.baseToolbox, phase.Tools)
+	maxIter := phase.MaxIterations
+	if maxIter <= 0 {
+		maxIter = deps.baseMaxIter
+	}
+	// Per-phase opts: a fixed step ceiling for this phase (the critic's dynamic
+	// ceiling is intentionally not propagated to phases) + the phase toolbox + the
+	// shared step observer (audit/steps/critic), on top of the shared opts
+	// (tool-error limits, compactor).
+	opts := append([]agent.Option{
+		agent.WithToolbox(toolbox),
+		agent.WithMaxSteps(maxIter),
+		agent.WithStepObserver(deps.stepObserver),
+	}, deps.sharedOpts...)
+	ag := agent.New(model, system, opts...)
+
+	res, runErr := runAgent(phaseCtx, ag, query, images, agent.WithSteer(deps.steer))
+	output := ""
+	if res != nil {
+		output = res.Output
+	}
+	// Budget/guard exhaustion leaves a usable partial transcript but an empty
+	// final answer; salvage the narrated work so the pipeline can carry it forward.
+	if runErr != nil && isPhaseBudgetExhaustion(runErr) {
+		if salvaged := salvagePhaseTranscript(res); salvaged != "" {
+			output = salvaged
+		}
+	}
+	return output, res, runErr
+}
+
+// phaseModel resolves the phase's model tier, returning the resolver's enriched
+// context (usage attribution) alongside the model. An empty tier or a resolution
+// failure falls back to the base model + the run context (WARN — visible, not
+// fatal). Returning the enriched ctx mirrors the single-loop path, which adopts
+// ctx = modelCtx, so a non-base-tier phase's calls are attributed correctly.
+func (e *Executor) phaseModel(ctx context.Context, deps phaseDeps, ra RunnableAgent, phase Phase) (context.Context, llm.Model) {
+	if phase.ModelTier == "" {
+		return ctx, deps.baseModel
+	}
+	modelCtx, m, err := e.cfg.Models(ctx, phase.ModelTier)
+	if err != nil || m == nil {
+		reason := "resolver returned a nil model"
+		if err != nil {
+			reason = err.Error()
+		}
+		slog.Warn("run: pipeline phase model resolve failed; using base model",
+			"agent", ra.Name, "phase", phase.Name, "tier", phase.ModelTier, "reason", reason)
+		return ctx, deps.baseModel
+	}
+	return modelCtx, m
+}
+
+// isPhaseBudgetExhaustion reports whether err is a soft budget/guard stop (the
+// loop hit its step cap or tripped a tool-error guard) — which leaves a usable
+// partial transcript — as opposed to a hard error (cancellation, model failure).
+func isPhaseBudgetExhaustion(err error) bool {
+	return errors.Is(err, agent.ErrMaxSteps) || errors.Is(err, agent.ErrToolLoop)
+}
+
+// maxSalvageBytes bounds a salvaged partial transcript so a long phase's narrated
+// reasoning doesn't blow up the next phase's prompt (the tail is the most recent,
+// most relevant reasoning). Matches mort's pipeline cap.
+const maxSalvageBytes = 8000
+
+// salvagePhaseTranscript reconstructs a best-effort phase output from a loop that
+// ended without a final answer: the assistant's narrated text across every step,
+// tail-trimmed to maxSalvageBytes on a rune boundary. Returns "" when the model
+// wrote no prose.
+func salvagePhaseTranscript(res *agent.Result) string {
+	if res == nil {
+		return ""
+	}
+	var b strings.Builder
+	for _, step := range res.Steps {
+		if step.Response == nil {
+			continue
+		}
+		if t := strings.TrimSpace(step.Response.Text()); t != "" {
+			if b.Len() > 0 {
+				b.WriteString("\n\n")
+			}
+			b.WriteString(t)
+		}
+	}
+	out := strings.TrimSpace(b.String())
+	if len(out) > maxSalvageBytes {
+		tail := out[len(out)-maxSalvageBytes:]
+		// Advance to the next rune boundary so the cut never splits a UTF-8 rune.
+		for len(tail) > 0 && !utf8.RuneStart(tail[0]) {
+			tail = tail[1:]
+		}
+		out = "...(earlier reasoning trimmed)...\n" + tail
+	}
+	return out
+}
+
+// multimodalUserMessage builds a user message from text + inline images. Shared
+// by the phase runner and runAgent so the image-folding lives in one place.
+// Empty text with images yields an image-only message (no empty text part).
+func multimodalUserMessage(text string, images []llm.ImagePart) llm.Message {
+	if len(images) == 0 {
+		return llm.UserText(text)
+	}
+	parts := make([]llm.Part, 0, len(images)+1)
+	if strings.TrimSpace(text) != "" {
+		parts = append(parts, llm.Text(text))
+	}
+	for _, img := range images {
+		parts = append(parts, img)
+	}
+	return llm.UserParts(parts...)
+}
+
+// expandPhaseTemplate applies Go text/template substitution to a phase prompt,
+// replacing {{.Query}} with the original query and {{.<PhaseName>}} with a prior
+// phase's output. On a parse/execute error it logs a WARN and returns the
+// template unchanged (best-effort, non-fatal) so a misconfigured prompt is
+// visible rather than silently masked.
+func expandPhaseTemplate(tmpl, query string, priorOutputs map[string]string) string {
+	t, err := template.New("phase").Option("missingkey=zero").Parse(tmpl)
+	if err != nil {
+		slog.Warn("run: pipeline phase template parse failed; using it unexpanded", "error", err)
+		return tmpl
+	}
+	data := map[string]string{"Query": query}
+	for k, v := range priorOutputs {
+		data[k] = v
+	}
+	var buf bytes.Buffer
+	if err := t.Execute(&buf, data); err != nil {
+		slog.Warn("run: pipeline phase template execute failed; using it unexpanded", "error", err)
+		return tmpl
+	}
+	return buf.String()
+}
+
+// filterToolbox returns a toolbox restricted to the named tools (preserving
+// palette order). Empty names = the full palette (the base toolbox is returned
+// as-is — it is read-only during a run, like the single-loop path). Unknown names
+// are skipped with a WARN — a typo'd phase tool list should not abort a run.
+func filterToolbox(box *llm.Toolbox, names []string) *llm.Toolbox {
+	if len(names) == 0 {
+		return box
+	}
+	out := llm.NewToolbox(box.Name())
+	for _, name := range names {
+		t, ok := box.Get(name)
+		if !ok {
+			slog.Warn("run: pipeline phase references unknown tool; skipping", "tool", name)
+			continue
+		}
+		if err := out.Add(t); err != nil {
+			slog.Warn("run: pipeline phase tool duplicated; skipping", "tool", name, "error", err)
+		}
+	}
+	return out
+}
+
+// addUsage sums two llm.Usage tallies field-by-field so a phased run reports the
+// total tokens across all phases. NOTE: if llm.Usage gains a field, add it here
+// too — the audit recorder (rec) is the authoritative per-run token source, this
+// is the secondary Result.Usage roll-up.
+func addUsage(a, b llm.Usage) llm.Usage {
+	a.InputTokens += b.InputTokens
+	a.OutputTokens += b.OutputTokens
+	a.CacheReadTokens += b.CacheReadTokens
+	a.CacheWriteTokens += b.CacheWriteTokens
+	a.ReasoningTokens += b.ReasoningTokens
+	return a
+}

run/phases_test.go

@@ -0,0 +1,278 @@
+package run
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+
+	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
+	"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
+
+	"gitea.stevedudenhoeffer.com/steve/executus/tool"
+)
+
+// phaseProvider builds a fake provider scripted with the given per-call steps
+// (consumed in order across every phase's model call) and a resolver over it,
+// returning both so a test can read back each call's request.
+func phaseProvider(t *testing.T, steps ...fake.Step) (ModelResolver, *fake.Provider) {
+	t.Helper()
+	fp := fake.New("fake")
+	fp.Enqueue("test-model", steps...)
+	m, err := fp.Model("test-model")
+	if err != nil {
+		t.Fatalf("fake model: %v", err)
+	}
+	return func(ctx context.Context, _ string) (context.Context, llm.Model, error) {
+		return ctx, m, nil
+	}, fp
+}
+
+// TestPhases_SequentialThreadsOutputs: phases run in order, each phase's output
+// is threaded into the next via {{.<PhaseName>}}, {{.Query}} reaches a phase, and
+// the final phase's output is the run output.
+func TestPhases_SequentialThreadsOutputs(t *testing.T) {
+	models, fp := phaseProvider(t,
+		fake.Reply("out-a"),
+		fake.Reply("out-b"),
+		fake.Reply("out-c"),
+	)
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models})
+
+	ra := RunnableAgent{
+		Name:      "pipeline",
+		ModelTier: "test-model",
+		Phases: []Phase{
+			{Name: "a", SystemPrompt: "Phase A instructions"},
+			{Name: "b", SystemPrompt: "B saw: {{.a}}"},
+			{Name: "c", SystemPrompt: "C saw: {{.b}} and query {{.Query}}"},
+		},
+	}
+	res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r", CallerID: "c"}, "QUERY-TEXT")
+	if res.Err != nil {
+		t.Fatalf("run error: %v", res.Err)
+	}
+	if res.Output != "out-c" {
+		t.Fatalf("final output = %q, want the LAST phase's output out-c", res.Output)
+	}
+	calls := fp.Calls()
+	if len(calls) != 3 {
+		t.Fatalf("want 3 model calls (one per phase), got %d", len(calls))
+	}
+	if got := calls[0].Request.System; got != "Phase A instructions" {
+		t.Errorf("phase a system = %q", got)
+	}
+	if got := calls[1].Request.System; got != "B saw: out-a" {
+		t.Errorf("phase b should see phase a's output threaded; system = %q", got)
+	}
+	if got := calls[2].Request.System; got != "C saw: out-b and query QUERY-TEXT" {
+		t.Errorf("phase c should see phase b's output + {{.Query}}; system = %q", got)
+	}
+}
+
+// TestPhases_OptionalFailureSubstitutesFallback: an Optional phase that errors
+// does not abort the pipeline — its FallbackMessage becomes its output and is
+// threaded into later phases, which still run.
+func TestPhases_OptionalFailureSubstitutesFallback(t *testing.T) {
+	models, fp := phaseProvider(t,
+		fake.Fail(errors.New("provider exploded")), // phase a fails
+		fake.Reply("out-b"),                        // phase b runs
+	)
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models})
+
+	ra := RunnableAgent{
+		Name:      "pipeline",
+		ModelTier: "test-model",
+		Phases: []Phase{
+			{Name: "a", SystemPrompt: "Phase A", Optional: true, FallbackMessage: "FALLBACK-A"},
+			{Name: "b", SystemPrompt: "B saw: {{.a}}"},
+		},
+	}
+	res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r", CallerID: "c"}, "Q")
+	if res.Err != nil {
+		t.Fatalf("optional-phase failure must not fail the run: %v", res.Err)
+	}
+	if res.Output != "out-b" {
+		t.Fatalf("final output = %q, want out-b", res.Output)
+	}
+	calls := fp.Calls()
+	if len(calls) != 2 {
+		t.Fatalf("want 2 calls (failed phase a + phase b), got %d", len(calls))
+	}
+	if got := calls[1].Request.System; got != "B saw: FALLBACK-A" {
+		t.Errorf("phase b should see the fallback threaded; system = %q", got)
+	}
+}
+
+// TestPhases_OptionalDoesNotSwallowCancellation: an Optional phase that fails
+// with a context cancellation must NOT be swallowed into its FallbackMessage —
+// the run genuinely ended (cancel/deadline/critic-kill) and must surface the
+// error so the run is classified cancelled/timeout/killed, not "ok".
+func TestPhases_OptionalDoesNotSwallowCancellation(t *testing.T) {
+	models, _ := phaseProvider(t, fake.Fail(context.Canceled))
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models})
+
+	ra := RunnableAgent{
+		Name:      "pipeline",
+		ModelTier: "test-model",
+		Phases: []Phase{
+			// IsRunFunc so the cancellation surfaces directly wrapped (%w).
+			{Name: "a", SystemPrompt: "Phase A", IsRunFunc: true, Optional: true, FallbackMessage: "FB"},
+		},
+	}
+	res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r", CallerID: "c"}, "Q")
+	if !errors.Is(res.Err, context.Canceled) {
+		t.Fatalf("Optional phase must NOT swallow a cancellation; res.Err = %v", res.Err)
+	}
+	if res.Output == "FB" {
+		t.Error("a cancelled run must not report the fallback message as output")
+	}
+}
+
+// TestPhases_DuplicateNamesBothRun: a fresh (non-resume) run with two phases
+// sharing a name must run BOTH — the resume-skip guard keys off a separate
+// resume set, not the live outputs map (which fills as phases run), so a phase
+// never skips a same-named sibling on a fresh run.
+func TestPhases_DuplicateNamesBothRun(t *testing.T) {
+	models, fp := phaseProvider(t, fake.Reply("first"), fake.Reply("second"))
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models})
+	ra := RunnableAgent{
+		Name: "p", ModelTier: "test-model",
+		Phases: []Phase{{Name: "x", SystemPrompt: "P1"}, {Name: "x", SystemPrompt: "P2"}},
+	}
+	res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r"}, "Q")
+	if res.Err != nil {
+		t.Fatalf("run error: %v", res.Err)
+	}
+	if n := len(fp.Calls()); n != 2 {
+		t.Fatalf("both same-named phases must run on a fresh run; got %d model calls", n)
+	}
+}
+
+// TestPhases_HardErrorAborts: a NON-optional phase that hits a hard error (not a
+// budget/step exhaustion) aborts the pipeline; later phases do not run.
+func TestPhases_HardErrorAborts(t *testing.T) {
+	boom := errors.New("model down")
+	models, fp := phaseProvider(t,
+		fake.Fail(boom),     // phase a (non-optional) fails hard
+		fake.Reply("out-b"), // must NOT be consumed
+	)
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models})
+
+	ra := RunnableAgent{
+		Name:      "pipeline",
+		ModelTier: "test-model",
+		Phases: []Phase{
+			{Name: "a", SystemPrompt: "Phase A"},
+			{Name: "b", SystemPrompt: "Phase B"},
+		},
+	}
+	res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r", CallerID: "c"}, "Q")
+	if res.Err == nil {
+		t.Fatal("a hard non-optional phase error must fail the run")
+	}
+	if !errors.Is(res.Err, boom) {
+		t.Errorf("run error %v should wrap the phase's model error", res.Err)
+	}
+	if n := len(fp.Calls()); n != 1 {
+		t.Errorf("pipeline must abort after phase a; got %d calls (phase b should not run)", n)
+	}
+}
+
+// TestPhases_IsRunFuncBareCall: an IsRunFunc phase produces output via a bare LLM
+// call and that output threads into a following loop phase.
+func TestPhases_IsRunFuncBareCall(t *testing.T) {
+	models, fp := phaseProvider(t,
+		fake.Reply("plan-output"), // IsRunFunc phase a
+		fake.Reply("final"),       // loop phase b
+	)
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models})
+
+	ra := RunnableAgent{
+		Name:      "pipeline",
+		ModelTier: "test-model",
+		Phases: []Phase{
+			{Name: "plan", SystemPrompt: "Make a plan for {{.Query}}", IsRunFunc: true},
+			{Name: "exec", SystemPrompt: "Execute: {{.plan}}"},
+		},
+	}
+	res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r", CallerID: "c"}, "do-thing")
+	if res.Err != nil {
+		t.Fatalf("run error: %v", res.Err)
+	}
+	if res.Output != "final" {
+		t.Fatalf("output = %q, want final", res.Output)
+	}
+	calls := fp.Calls()
+	if len(calls) != 2 {
+		t.Fatalf("want 2 calls, got %d", len(calls))
+	}
+	if got := calls[0].Request.System; got != "Make a plan for do-thing" {
+		t.Errorf("IsRunFunc phase system = %q", got)
+	}
+	if got := calls[1].Request.System; got != "Execute: plan-output" {
+		t.Errorf("exec phase should see the plan output threaded; system = %q", got)
+	}
+}
+
+// TestPhases_SystemHeaderAppliedPerPhase: the platform SystemHeader is prepended
+// to every phase's prompt (each phase keeps it).
+func TestPhases_SystemHeaderAppliedPerPhase(t *testing.T) {
+	models, fp := phaseProvider(t, fake.Reply("a"), fake.Reply("b"))
+	ex := New(Config{Registry: tool.NewRegistry(), Models: models, SystemHeader: "PLATFORM"})
+
+	ra := RunnableAgent{
+		Name:      "p",
+		ModelTier: "test-model",
+		Phases:    []Phase{{Name: "one", SystemPrompt: "P1"}, {Name: "two", SystemPrompt: "P2"}},
+	}
+	if res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r"}, "Q"); res.Err != nil {
+		t.Fatalf("run error: %v", res.Err)
+	}
+	for i, want := range []string{"PLATFORM\n\nP1", "PLATFORM\n\nP2"} {
+		if got := fp.Calls()[i].Request.System; got != want {
+			t.Errorf("phase %d system = %q, want %q", i, got, want)
+		}
+	}
+}
+
+// TestFilterToolbox: a named subset restricts the toolbox (preserving order);
+// empty names = the full palette; unknown names are skipped.
+func TestFilterToolbox(t *testing.T) {
+	box := llm.NewToolbox("base")
+	noop := func(context.Context, json.RawMessage) (any, error) { return "", nil }
+	for _, name := range []string{"alpha", "beta", "gamma"} {
+		if err := box.Add(llm.Tool{Name: name, Description: "d", Handler: noop}); err != nil {
+			t.Fatalf("add %s: %v", name, err)
+		}
+	}
+
+	full := filterToolbox(box, nil)
+	if len(full.Tools()) != 3 {
+		t.Errorf("nil names = full palette; got %d tools", len(full.Tools()))
+	}
+
+	sub := filterToolbox(box, []string{"gamma", "alpha", "nonexistent"})
+	names := make([]string, 0)
+	for _, tl := range sub.Tools() {
+		names = append(names, tl.Name)
+	}
+	if strings.Join(names, ",") != "gamma,alpha" {
+		t.Errorf("subset (order-preserving, unknown skipped) = %v, want [gamma alpha]", names)
+	}
+}
+
+// TestExpandPhaseTemplate: {{.Query}} + prior outputs substitute; a parse error
+// returns the template unchanged (best-effort).
+func TestExpandPhaseTemplate(t *testing.T) {
+	got := expandPhaseTemplate("q={{.Query}} a={{.a}}", "QQ", map[string]string{"a": "AA"})
+	if got != "q=QQ a=AA" {
+		t.Errorf("expand = %q", got)
+	}
+	// Malformed template → returned unchanged.
+	bad := "{{.Unclosed"
+	if expandPhaseTemplate(bad, "QQ", nil) != bad {
+		t.Errorf("malformed template should pass through unchanged")
+	}
+}

run/ports.go

@@ -33,9 +33,10 @@ type Ports struct {
 	Budget Budget
 	// Critic optionally monitors a long run for hangs/runaways. nil = none.
 	Critic Critic
-	// Checkpointer persists resumable progress for durable recovery. nil = no
+	// Checkpointer mints a per-run Checkpointer for durable recovery (it decides
-	// checkpointing (a run interrupted by shutdown is simply lost).
+	// per run whether the run is durable). nil = no checkpointing (a run
-	Checkpointer Checkpointer
+	// interrupted by shutdown is simply lost).
+	Checkpointer CheckpointerFactory
 	// Palette resolves SkillPalette / SubAgentPalette entries into delegation
 	// tools (skill__<name> / agent__<name>). nil = those entries are inert.
 	Palette PaletteSource
@@ -66,7 +67,9 @@ type RunInfo struct {
 	Name        string
 	CallerID    string
 	ChannelID   string
+	GuildID     string // the originating guild/server id (empty for DMs/triggers)
 	ParentRunID string
+	ModelTier   string // the run's resolved base tier (for checkpoint re-dispatch)
 	Inputs      map[string]any
 	StartedAt   time.Time
 	// MaxIterations is the run's base tool-dispatch step ceiling, so a critic can
@@ -172,6 +175,16 @@ type CriticHandle interface {
 
 // --- Checkpointer ---
 
+// CheckpointerFactory decides, per run, whether the run is durable and (if so)
+// mints the per-run Checkpointer that records its progress. It returns (nil, nil)
+// for a non-durable run (the common short-run case — no checkpointing overhead).
+// A storage error should be logged and degraded to (nil, nil) so a failing
+// checkpoint store never fails the run. Mirrors mort's
+// agentexec.CheckpointerFactory.
+type CheckpointerFactory interface {
+	Begin(ctx context.Context, info RunInfo) (Checkpointer, error)
+}
+
 // Checkpointer persists a run's resumable progress for durable recovery.
 // Mirrors mort's agentexec.RunCheckpointer.
 type Checkpointer interface {
@@ -184,11 +197,24 @@ type Checkpointer interface {
 	Fail(ctx context.Context, err error) error
 }
 
-// RunCheckpointState is the resumable snapshot a Checkpointer persists. Kept
+// RunCheckpointState is the resumable snapshot a Checkpointer persists.
-// minimal here; the executor extends what it records during the merge.
 type RunCheckpointState struct {
+	// Messages is the running transcript of a SINGLE-LOOP run (grows each step;
+	// resumed via WithHistory). nil for multi-phase runs — phase recovery is
+	// boundary-granular (see CompletedPhases), not mid-phase transcript.
 	Messages  []llm.Message
 	Iteration int
+	// CompletedPhases is set only for multi-phase runs: the outputs of phases
+	// already finished, in phase order, so a resumed run skips them and re-runs
+	// the interrupted phase from its start. nil for single-loop runs.
+	CompletedPhases []PhaseOutput
+}
+
+// PhaseOutput is one completed pipeline phase's name and output text, recorded in
+// a checkpoint so a resumed multi-phase run can skip already-finished phases.
+type PhaseOutput struct {
+	Name   string
+	Output string
 }
 
 // --- PaletteSource ---