Compare commits
7 Commits
c071ed4996
...
v0.1.5
| Author | SHA1 | Date | |
|---|---|---|---|
| 2a43210f38 | |||
| 79ce833dd7 | |||
| cb4c612461 | |||
| 5b5ee4148e | |||
| 31f9078915 | |||
| 38d656ec71 | |||
| 899059a791 |
@@ -4,9 +4,9 @@
|
||||
// run.Ports.Checkpointer.
|
||||
//
|
||||
// Mort backs CheckpointStore with its durable-job table; Memory() is the
|
||||
// zero-dependency default; contrib/store can add a SQLite one. NOTE: the
|
||||
// executor's call into run.Ports.Checkpointer is a P2 follow-up — this battery
|
||||
// provides the seam + impls ahead of that wiring.
|
||||
// zero-dependency default; contrib/store can add a SQLite one. The executor calls
|
||||
// run.Ports.Checkpointer (a CheckpointerFactory) during the run loop; NewFactory
|
||||
// wires this battery into that seam.
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/run"
|
||||
)
|
||||
|
||||
// RunCheckpointMeta is the run attribution needed to resume a run from scratch
|
||||
@@ -33,9 +35,9 @@ type RunCheckpointMeta struct {
|
||||
// RunCheckpoint is one persisted snapshot of a run's resumable progress.
|
||||
type RunCheckpoint struct {
|
||||
Meta RunCheckpointMeta
|
||||
Messages []llm.Message // conversation so far
|
||||
Messages []llm.Message // conversation so far (single-loop runs)
|
||||
Iteration int // completed agent-loop iterations
|
||||
ActivePhase string // current phase name (multi-phase agents); "" otherwise
|
||||
CompletedPhases []run.PhaseOutput // finished phases, in order (multi-phase agents)
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
|
||||
@@ -57,6 +57,7 @@ func (h *handle) Save(ctx context.Context, st run.RunCheckpointState) error {
|
||||
Meta: h.meta,
|
||||
Messages: st.Messages,
|
||||
Iteration: st.Iteration,
|
||||
CompletedPhases: st.CompletedPhases,
|
||||
UpdatedAt: now,
|
||||
}); err != nil {
|
||||
return err
|
||||
@@ -81,3 +82,40 @@ var _ run.Checkpointer = noop{}
|
||||
func (noop) Save(context.Context, run.RunCheckpointState) error { return nil }
|
||||
func (noop) Complete(context.Context) error { return nil }
|
||||
func (noop) Fail(context.Context, error) error { return nil }
|
||||
|
||||
// factory is a run.CheckpointerFactory that mints a per-run handle over store,
|
||||
// deriving the per-run meta from the kernel's RunInfo. It is the battery's glue
|
||||
// for the Ports.Checkpointer (factory) seam: every run becomes durable (the
|
||||
// store persists snapshots; a host wanting lazy/short-run skipping uses its own
|
||||
// factory, as mort does over its durable-job table).
|
||||
type factory struct {
|
||||
store CheckpointStore
|
||||
throttle time.Duration
|
||||
}
|
||||
|
||||
var _ run.CheckpointerFactory = (*factory)(nil)
|
||||
|
||||
// NewFactory returns a run.CheckpointerFactory backed by store: each run gets a
|
||||
// per-run Checkpointer (throttled to at most once per throttle). A nil store
|
||||
// yields factory.Begin returning a no-op Checkpointer.
|
||||
func NewFactory(store CheckpointStore, throttle time.Duration) run.CheckpointerFactory {
|
||||
return &factory{store: store, throttle: throttle}
|
||||
}
|
||||
|
||||
// Begin mints the per-run Checkpointer. The prompt is read from
|
||||
// info.Inputs["prompt"] when present so a recovered run can re-dispatch.
|
||||
func (f *factory) Begin(_ context.Context, info run.RunInfo) (run.Checkpointer, error) {
|
||||
prompt, _ := info.Inputs["prompt"].(string)
|
||||
meta := RunCheckpointMeta{
|
||||
RunID: info.RunID,
|
||||
AgentID: info.SubjectID,
|
||||
AgentName: info.Name,
|
||||
CallerID: info.CallerID,
|
||||
ChannelID: info.ChannelID,
|
||||
GuildID: info.GuildID,
|
||||
Prompt: prompt,
|
||||
ModelTier: info.ModelTier,
|
||||
ParentRunID: info.ParentRunID,
|
||||
}
|
||||
return New(f.store, meta, f.throttle, nil /* now defaults to time.Now */), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// Durable-recovery plumbing for the executor. The Checkpointer port (set via
|
||||
// Ports.Checkpointer, a CheckpointerFactory) persists a run's resumable progress
|
||||
// during the loop; on boot a host re-dispatches an interrupted run through the
|
||||
// executor with a ResumeState (the saved transcript / completed phases) so it
|
||||
// CONTINUES rather than restarting, reusing the SAME durable record via an
|
||||
// existing Checkpointer. Both are carried into Run via the context (mirrors
|
||||
// mort's agentexec.WithResumeState / WithExistingCheckpointer).
|
||||
|
||||
// ResumeState carries a recovered run's prior progress into Run so the run
|
||||
// continues instead of restarting. The host's recovery path sets it via
|
||||
// WithResumeState; the executor reads it:
|
||||
// - single-loop: History seeds the saved transcript (the run continues).
|
||||
// - multi-phase: CompletedPhases are skipped; the interrupted phase re-runs
|
||||
// from its start (boundary-granular — there is no mid-phase transcript
|
||||
// resume, so History is unused for multi-phase runs).
|
||||
type ResumeState struct {
|
||||
History []llm.Message // single-loop transcript (unused for multi-phase)
|
||||
CompletedPhases []PhaseOutput // multi-phase: outputs of finished phases, in order
|
||||
}
|
||||
|
||||
type resumeStateKey struct{}
|
||||
|
||||
// WithResumeState carries a recovered run's prior progress into Run.
|
||||
func WithResumeState(ctx context.Context, rs *ResumeState) context.Context {
|
||||
return context.WithValue(ctx, resumeStateKey{}, rs)
|
||||
}
|
||||
|
||||
func resumeStateFromContext(ctx context.Context) *ResumeState {
|
||||
rs, _ := ctx.Value(resumeStateKey{}).(*ResumeState)
|
||||
return rs
|
||||
}
|
||||
|
||||
type existingCheckpointerKey struct{}
|
||||
|
||||
// WithExistingCheckpointer carries a pre-existing Checkpointer into Run so a
|
||||
// recovery re-run reuses the SAME durable record (the executor uses it instead of
|
||||
// calling Ports.Checkpointer.Begin).
|
||||
func WithExistingCheckpointer(ctx context.Context, cp Checkpointer) context.Context {
|
||||
return context.WithValue(ctx, existingCheckpointerKey{}, cp)
|
||||
}
|
||||
|
||||
func existingCheckpointerFromContext(ctx context.Context) Checkpointer {
|
||||
cp, _ := ctx.Value(existingCheckpointerKey{}).(Checkpointer)
|
||||
return cp
|
||||
}
|
||||
|
||||
// checkpointOutcome is the finalize decision for a durable run.
|
||||
type checkpointOutcome int
|
||||
|
||||
const (
|
||||
checkpointComplete checkpointOutcome = iota
|
||||
checkpointLeaveRunning
|
||||
checkpointFail
|
||||
)
|
||||
|
||||
// classifyCheckpointOutcome maps (run error, cancellation cause) to the durable
|
||||
// finalize action: success clears the checkpoint (Complete); a shutdown-caused
|
||||
// cancellation leaves the record so boot recovery picks it up (neither
|
||||
// Complete nor Fail); anything else (model error, tool loop, the run's own
|
||||
// deadline, a critic kill, a caller cancel) is terminal (Fail). Mirrors mort's
|
||||
// agentexec.classifyCheckpointOutcome.
|
||||
func classifyCheckpointOutcome(runErr, cause error) checkpointOutcome {
|
||||
switch {
|
||||
case runErr == nil:
|
||||
return checkpointComplete
|
||||
case errors.Is(cause, ErrShutdown):
|
||||
return checkpointLeaveRunning
|
||||
default:
|
||||
return checkpointFail
|
||||
}
|
||||
}
|
||||
|
||||
// finalizeCheckpoint applies the outcome to the per-run checkpointer (nil-safe).
|
||||
// Runs on a detached context so a cancelled run still records its terminal state.
|
||||
// Complete/Fail errors are best-effort but logged (a stale record would only
|
||||
// cause a wasteful boot-recovery retry, not data loss).
|
||||
func finalizeCheckpoint(ctx context.Context, cp Checkpointer, runErr error, cause error) {
|
||||
if cp == nil {
|
||||
return
|
||||
}
|
||||
switch classifyCheckpointOutcome(runErr, cause) {
|
||||
case checkpointComplete:
|
||||
if err := cp.Complete(detach(ctx)); err != nil {
|
||||
slog.Warn("run: checkpoint Complete failed", "error", err)
|
||||
}
|
||||
case checkpointFail:
|
||||
if err := cp.Fail(detach(ctx), runErr); err != nil {
|
||||
slog.Warn("run: checkpoint Fail failed", "error", err)
|
||||
}
|
||||
case checkpointLeaveRunning:
|
||||
// Interrupted by shutdown: leave the record for boot recovery.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/tool"
|
||||
)
|
||||
|
||||
// fakeCheckpointer records every Save state + whether Complete/Fail fired.
|
||||
type fakeCheckpointer struct {
|
||||
saves []RunCheckpointState
|
||||
completed bool
|
||||
failed bool
|
||||
failErr error
|
||||
}
|
||||
|
||||
func (c *fakeCheckpointer) Save(_ context.Context, st RunCheckpointState) error {
|
||||
c.saves = append(c.saves, st)
|
||||
return nil
|
||||
}
|
||||
func (c *fakeCheckpointer) Complete(context.Context) error { c.completed = true; return nil }
|
||||
func (c *fakeCheckpointer) Fail(_ context.Context, err error) error {
|
||||
c.failed = true
|
||||
c.failErr = err
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeCheckpointFactory hands out one fakeCheckpointer and records the RunInfo.
|
||||
type fakeCheckpointFactory struct {
|
||||
cp *fakeCheckpointer
|
||||
info RunInfo
|
||||
}
|
||||
|
||||
func (f *fakeCheckpointFactory) Begin(_ context.Context, info RunInfo) (Checkpointer, error) {
|
||||
f.info = info
|
||||
return f.cp, nil
|
||||
}
|
||||
|
||||
// TestClassifyCheckpointOutcome covers the finalize decision matrix.
|
||||
func TestClassifyCheckpointOutcome(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
cause error
|
||||
want checkpointOutcome
|
||||
}{
|
||||
{"success", nil, nil, checkpointComplete},
|
||||
{"shutdown", context.Canceled, ErrShutdown, checkpointLeaveRunning},
|
||||
{"critic-kill", context.Canceled, ErrCriticKill, checkpointFail},
|
||||
{"deadline", context.DeadlineExceeded, context.DeadlineExceeded, checkpointFail},
|
||||
{"model-error", errors.New("boom"), nil, checkpointFail},
|
||||
{"caller-cancel", context.Canceled, context.Canceled, checkpointFail},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := classifyCheckpointOutcome(tc.err, tc.cause); got != tc.want {
|
||||
t.Errorf("%s: classifyCheckpointOutcome = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckpoint_SingleLoopSaveAndComplete: a durable single-loop run gets a
|
||||
// per-run checkpointer (Begin), Saves its transcript each step, and Completes on
|
||||
// success (clearing the checkpoint). The RunInfo carries the resume meta.
|
||||
func TestCheckpoint_SingleLoopSaveAndComplete(t *testing.T) {
|
||||
models, _ := phaseProvider(t, fake.Reply("done"))
|
||||
cp := &fakeCheckpointer{}
|
||||
f := &fakeCheckpointFactory{cp: cp}
|
||||
ex := New(Config{Registry: tool.NewRegistry(), Models: models, Ports: Ports{Checkpointer: f}})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ID: "a1", Name: "boss", ModelTier: "test-model"},
|
||||
tool.Invocation{RunID: "run-x", CallerID: "steve", ChannelID: "chan", GuildID: "g", SkillInputs: map[string]any{"prompt": "go"}},
|
||||
"go")
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if f.info.RunID != "run-x" || f.info.SubjectID != "a1" || f.info.ModelTier != "test-model" || f.info.GuildID != "g" {
|
||||
t.Errorf("Begin RunInfo missing resume meta: %+v", f.info)
|
||||
}
|
||||
if len(cp.saves) == 0 {
|
||||
t.Error("expected at least one checkpoint Save during the run")
|
||||
} else if len(cp.saves[len(cp.saves)-1].Messages) == 0 {
|
||||
t.Error("checkpoint Save should carry the running transcript")
|
||||
}
|
||||
if !cp.completed {
|
||||
t.Error("a successful run must Complete (clear) its checkpoint")
|
||||
}
|
||||
if cp.failed {
|
||||
t.Error("a successful run must NOT Fail its checkpoint")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckpoint_TerminalErrorFails: a run that errors (not shutdown) Fails its
|
||||
// checkpoint (clears it — not a recovery candidate).
|
||||
func TestCheckpoint_TerminalErrorFails(t *testing.T) {
|
||||
models, _ := phaseProvider(t, fake.Fail(errors.New("model down")))
|
||||
cp := &fakeCheckpointer{}
|
||||
ex := New(Config{Registry: tool.NewRegistry(), Models: models, Ports: Ports{Checkpointer: &fakeCheckpointFactory{cp: cp}}})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ID: "a1", ModelTier: "test-model"},
|
||||
tool.Invocation{RunID: "r", CallerID: "c", SkillInputs: map[string]any{"prompt": "go"}}, "go")
|
||||
if res.Err == nil {
|
||||
t.Fatal("expected a run error")
|
||||
}
|
||||
if !cp.failed {
|
||||
t.Error("a terminal (non-shutdown) error must Fail the checkpoint")
|
||||
}
|
||||
if cp.completed {
|
||||
t.Error("a failed run must NOT Complete its checkpoint")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckpoint_ResumeSeedsHistory: a run carrying a ResumeState seeds the saved
|
||||
// transcript as the model's opening messages (continues) instead of the input.
|
||||
func TestCheckpoint_ResumeSeedsHistory(t *testing.T) {
|
||||
models, fp := phaseProvider(t, fake.Reply("continued"))
|
||||
history := []llm.Message{llm.UserText("prior turn 1"), llm.AssistantText("prior answer 1")}
|
||||
ctx := WithResumeState(context.Background(), &ResumeState{History: history})
|
||||
|
||||
ex := New(Config{Registry: tool.NewRegistry(), Models: models})
|
||||
res := ex.Run(ctx,
|
||||
RunnableAgent{ID: "a1", ModelTier: "test-model"},
|
||||
tool.Invocation{RunID: "r", CallerID: "c", SkillInputs: map[string]any{"prompt": "ignored-on-resume"}}, "ignored-on-resume")
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
got := fp.Calls()[0].Request.Messages
|
||||
if len(got) != len(history) {
|
||||
t.Fatalf("resume should seed the saved %d-message transcript, got %d messages", len(history), len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckpoint_PhaseBoundarySavesCompleted: a durable multi-phase run records
|
||||
// the completed phases at each boundary, growing the list, and Completes on
|
||||
// success.
|
||||
func TestCheckpoint_PhaseBoundarySavesCompleted(t *testing.T) {
|
||||
models, _ := phaseProvider(t, fake.Reply("out-a"), fake.Reply("out-b"))
|
||||
cp := &fakeCheckpointer{}
|
||||
ex := New(Config{Registry: tool.NewRegistry(), Models: models, Ports: Ports{Checkpointer: &fakeCheckpointFactory{cp: cp}}})
|
||||
|
||||
ra := RunnableAgent{
|
||||
ID: "p", ModelTier: "test-model",
|
||||
Phases: []Phase{{Name: "a", SystemPrompt: "A"}, {Name: "b", SystemPrompt: "B"}},
|
||||
}
|
||||
if res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r", CallerID: "c"}, "Q"); res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
// The final phase-boundary Save must list both completed phases.
|
||||
var lastPhaseSave *RunCheckpointState
|
||||
for i := range cp.saves {
|
||||
if len(cp.saves[i].CompletedPhases) > 0 {
|
||||
lastPhaseSave = &cp.saves[i]
|
||||
}
|
||||
}
|
||||
if lastPhaseSave == nil || len(lastPhaseSave.CompletedPhases) != 2 {
|
||||
t.Fatalf("expected a phase-boundary Save listing 2 completed phases; saves=%+v", cp.saves)
|
||||
}
|
||||
if !cp.completed {
|
||||
t.Error("a successful phased run must Complete its checkpoint")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckpoint_ResumeSkipsCompletedPhases: a resumed multi-phase run skips
|
||||
// phases already in ResumeState.CompletedPhases (only the remaining phase calls
|
||||
// the model) and threads their outputs into the remaining phase's template.
|
||||
func TestCheckpoint_ResumeSkipsCompletedPhases(t *testing.T) {
|
||||
models, fp := phaseProvider(t, fake.Reply("out-b")) // ONLY phase b should call the model
|
||||
ctx := WithResumeState(context.Background(), &ResumeState{
|
||||
CompletedPhases: []PhaseOutput{{Name: "a", Output: "saved-a"}},
|
||||
})
|
||||
ex := New(Config{Registry: tool.NewRegistry(), Models: models})
|
||||
|
||||
ra := RunnableAgent{
|
||||
ID: "p", ModelTier: "test-model",
|
||||
Phases: []Phase{
|
||||
{Name: "a", SystemPrompt: "A"},
|
||||
{Name: "b", SystemPrompt: "B saw {{.a}}"},
|
||||
},
|
||||
}
|
||||
res := ex.Run(ctx, ra, tool.Invocation{RunID: "r", CallerID: "c"}, "Q")
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "out-b" {
|
||||
t.Fatalf("output = %q, want out-b", res.Output)
|
||||
}
|
||||
calls := fp.Calls()
|
||||
if len(calls) != 1 {
|
||||
t.Fatalf("only the un-completed phase b should call the model; got %d calls", len(calls))
|
||||
}
|
||||
if calls[0].Request.System != "B saw saved-a" {
|
||||
t.Errorf("resumed phase b should see the completed phase a's saved output; system = %q", calls[0].Request.System)
|
||||
}
|
||||
}
|
||||
+19
-7
@@ -22,6 +22,14 @@ type criticBinding struct {
|
||||
h CriticHandle
|
||||
}
|
||||
|
||||
// criticOwnsDeadline reports whether a critic is configured AND this run enables
|
||||
// it — the single predicate that decides the two-tier-timeout path. Used by BOTH
|
||||
// Run (to choose the generous runaway ceiling over the literal MaxRuntime cap) and
|
||||
// startCritic (the arm/no-op gate), so the two can never drift.
|
||||
func (e *Executor) criticOwnsDeadline(ra RunnableAgent) bool {
|
||||
return e.cfg.Ports.Critic != nil && ra.Critic.Enabled
|
||||
}
|
||||
|
||||
// startCritic begins critic monitoring for this run when one is configured and
|
||||
// the agent enables it. It launches a goroutine that cancels runCtx (via
|
||||
// cancelCause) the moment the critic's hard deadline passes — the critic may
|
||||
@@ -31,16 +39,20 @@ type criticBinding struct {
|
||||
// "killed"); when the backstop simply expired, it is context.DeadlineExceeded (→
|
||||
// "timeout"). Returns (nil, no-op stop) when there is no critic. The caller MUST
|
||||
// defer the returned stop.
|
||||
func (e *Executor) startCritic(runCtx context.Context, cancelCause context.CancelCauseFunc, ra RunnableAgent, info RunInfo) (*criticBinding, func()) {
|
||||
//
|
||||
// softTrigger is the run's resolved MaxRuntime: for a critic-owned run MaxRuntime
|
||||
// is the soft wake (mort's two-tier semantics — the critic first reviews once the
|
||||
// run exceeds its nominal budget, and its backstop = softTrigger × multiplier).
|
||||
// The caller (Run) always passes the resolved MaxRuntime, which withFallbacks
|
||||
// guarantees is > 0, so no fallback is needed here. (A non-positive soft would make
|
||||
// the host Monitor return no handle, and Run's unsupervised-run failsafe then bounds
|
||||
// the run at MaxRuntime — so even that impossible case stays bounded.)
|
||||
func (e *Executor) startCritic(runCtx context.Context, cancelCause context.CancelCauseFunc, ra RunnableAgent, info RunInfo, softTrigger time.Duration) (*criticBinding, func()) {
|
||||
noop := func() {}
|
||||
if e.cfg.Ports.Critic == nil || !ra.Critic.Enabled {
|
||||
if !e.criticOwnsDeadline(ra) {
|
||||
return nil, noop
|
||||
}
|
||||
soft := e.cfg.Defaults.CriticSoftTimeout
|
||||
if soft <= 0 {
|
||||
soft = 90 * time.Second // defensive: withFallbacks normally guarantees >0
|
||||
}
|
||||
h := e.cfg.Ports.Critic.Monitor(runCtx, info, soft)
|
||||
h := e.cfg.Ports.Critic.Monitor(runCtx, info, softTrigger)
|
||||
if h == nil {
|
||||
return nil, noop
|
||||
}
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package run_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/run"
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/tool"
|
||||
)
|
||||
|
||||
// slowToolInvocation builds an Invocation whose session factory adds a "slow"
|
||||
// tool that sleeps for d (respecting ctx). The model script calls it once, then
|
||||
// answers — so the run's wall-clock is dominated by d, letting a test set a tiny
|
||||
// MaxRuntime and observe whether MaxRuntime hard-cancels the run.
|
||||
func slowToolInvocation(runID string, d time.Duration) tool.Invocation {
|
||||
slow := llm.DefineTool("slow", "sleeps for a while",
|
||||
func(ctx context.Context, _ struct{}) (any, error) {
|
||||
select {
|
||||
case <-time.After(d):
|
||||
return "ok", nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
})
|
||||
return tool.Invocation{
|
||||
RunID: runID,
|
||||
SessionToolFactory: func(_ tool.AgentSession) tool.SessionTools {
|
||||
return tool.SessionTools{Tools: []llm.Tool{slow}}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func slowModel() llm.Model {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c1", Name: "slow", Arguments: []byte(`{}`)}}}),
|
||||
fake.Reply("done"),
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
return m
|
||||
}
|
||||
|
||||
// TestNoCritic_MaxRuntimeIsHardCap: the legacy contract is preserved — without a
|
||||
// critic, MaxRuntime is a literal WithTimeout that kills a run whose work outlasts
|
||||
// it. The slow tool (200ms) outlasts MaxRuntime (20ms), so runCtx cancels mid-tool
|
||||
// and the run ends in error (timeout).
|
||||
func TestNoCritic_MaxRuntimeIsHardCap(t *testing.T) {
|
||||
m := slowModel()
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 5, MaxRuntime: 20 * time.Millisecond},
|
||||
slowToolInvocation("r", 200*time.Millisecond), "go")
|
||||
if res.Err == nil {
|
||||
t.Fatalf("non-critic run should hard-timeout at MaxRuntime; got output=%q err=nil", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCriticOwnsDeadline_SurvivesPastMaxRuntime: the fix — when the critic owns the
|
||||
// deadline (Ports.Critic set + Critic.Enabled), MaxRuntime becomes the SOFT trigger
|
||||
// and is NOT a hard cap. The fake critic exposes no hard deadline (Deadline()==zero,
|
||||
// no kill), so the only hard ceiling is CriticAbsoluteMax (10s here). The slow tool
|
||||
// (200ms) outlasts the tiny MaxRuntime (20ms) but the run completes — proving the
|
||||
// old agentexec two-tier semantics are restored.
|
||||
func TestCriticOwnsDeadline_SurvivesPastMaxRuntime(t *testing.T) {
|
||||
m := slowModel()
|
||||
h := &fakeCriticHandle{} // Deadline()==zero → no hard deadline, no kill
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: run.Ports{Critic: &fakeCritic{h: h}},
|
||||
Defaults: run.Defaults{CriticAbsoluteMax: 10 * time.Second},
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "watched", ModelTier: "m", MaxIterations: 5, MaxRuntime: 20 * time.Millisecond,
|
||||
Critic: run.CriticConfig{Enabled: true}},
|
||||
slowToolInvocation("r", 200*time.Millisecond), "go")
|
||||
if res.Err != nil {
|
||||
t.Fatalf("critic-owned run must survive past MaxRuntime (soft trigger); got err=%v", res.Err)
|
||||
}
|
||||
if res.Output != "done" {
|
||||
t.Errorf("output = %q, want %q", res.Output, "done")
|
||||
}
|
||||
}
|
||||
|
||||
// capturingCritic records the soft trigger the executor passes to Monitor.
|
||||
type capturingCritic struct {
|
||||
mu sync.Mutex
|
||||
soft time.Duration
|
||||
h run.CriticHandle
|
||||
}
|
||||
|
||||
func (c *capturingCritic) Monitor(_ context.Context, _ run.RunInfo, soft time.Duration) run.CriticHandle {
|
||||
c.mu.Lock()
|
||||
c.soft = soft
|
||||
c.mu.Unlock()
|
||||
return c.h
|
||||
}
|
||||
|
||||
// TestCriticSoftTriggerIsMaxRuntime: the soft trigger handed to the host critic is
|
||||
// the run's resolved MaxRuntime (mort's two-tier model — the critic first wakes once
|
||||
// the run exceeds its nominal budget), not some global/default value.
|
||||
func TestCriticSoftTriggerIsMaxRuntime(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("done"))
|
||||
m, _ := fp.Model("m")
|
||||
cc := &capturingCritic{h: &fakeCriticHandle{}}
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: run.Ports{Critic: cc},
|
||||
})
|
||||
const wantSoft = 7 * time.Minute
|
||||
ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxRuntime: wantSoft, Critic: run.CriticConfig{Enabled: true}},
|
||||
tool.Invocation{RunID: "r"}, "go")
|
||||
cc.mu.Lock()
|
||||
got := cc.soft
|
||||
cc.mu.Unlock()
|
||||
if got != wantSoft {
|
||||
t.Errorf("soft trigger = %v, want the agent's MaxRuntime %v", got, wantSoft)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCriticOwnsDeadline_NilHandleFallsBackToMaxRuntime: the agent enables the
|
||||
// critic but the host Monitor returns NO handle (nil) — there is no deadline-watch,
|
||||
// so the run is unsupervised. It must fall back to the nominal MaxRuntime hard cap
|
||||
// (the slow 200ms tool outlasts the 20ms MaxRuntime → the run errors), NOT run free
|
||||
// up to the generous CriticAbsoluteMax runaway ceiling.
|
||||
func TestCriticOwnsDeadline_NilHandleFallsBackToMaxRuntime(t *testing.T) {
|
||||
m := slowModel()
|
||||
cc := &capturingCritic{} // h is the nil interface → Monitor returns a nil handle
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: run.Ports{Critic: cc},
|
||||
Defaults: run.Defaults{CriticAbsoluteMax: time.Hour}, // generous ceiling; must NOT be what bounds the run
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 5, MaxRuntime: 20 * time.Millisecond,
|
||||
Critic: run.CriticConfig{Enabled: true}},
|
||||
slowToolInvocation("r", 200*time.Millisecond), "go")
|
||||
if res.Err == nil {
|
||||
t.Fatalf("critic-enabled run with a nil Monitor handle must fall back to the MaxRuntime hard cap; got output=%q err=nil", res.Output)
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -61,8 +61,8 @@ func TestCriticRaisesStepCeiling(t *testing.T) {
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: run.Ports{Critic: &fakeCritic{h: h}},
|
||||
// large soft timeout so the deadline-watch never interferes in the test
|
||||
Defaults: run.Defaults{CriticSoftTimeout: time.Hour},
|
||||
// The fake handle's Deadline() is zero (no hard deadline), so the
|
||||
// deadline-watch never interferes regardless of the soft trigger.
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 1, Critic: run.CriticConfig{Enabled: true}},
|
||||
|
||||
+152
-26
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
|
||||
@@ -28,7 +29,17 @@ type Defaults struct {
|
||||
MaxConsecutiveToolErrors int // loop guard; default 3
|
||||
MaxSameToolCallRepeats int // retry-storm guard; default 3
|
||||
CompactionThresholdRatio float64 // fraction of model context to compact at; default 0.7
|
||||
CriticSoftTimeout time.Duration // idle window before the critic wakes; default 90s
|
||||
// CriticAbsoluteMax is the RUNAWAY ceiling for a critic-OWNED run (Ports.Critic
|
||||
// set AND the agent enables it). For such a run MaxRuntime is the SOFT trigger,
|
||||
// not a hard cap, and the critic's own extendable backstop is the normal
|
||||
// deadline. This ceiling exists ONLY to stop a critic that never advances its
|
||||
// deadline (a broken host handle) from running forever, so it is deliberately
|
||||
// set FAR beyond any realistic backstop (default 24h): the host clamps its own
|
||||
// backstop to a much smaller absolute max (e.g. a 6h host convar), so the ceiling
|
||||
// never pre-empts a healthy supervised run. Keep it well above the host's
|
||||
// absolute max. Never shorter than the run's MaxRuntime. Non-critic runs ignore
|
||||
// it (they keep the literal MaxRuntime kill).
|
||||
CriticAbsoluteMax time.Duration
|
||||
}
|
||||
|
||||
func (d Defaults) withFallbacks() Defaults {
|
||||
@@ -50,8 +61,8 @@ func (d Defaults) withFallbacks() Defaults {
|
||||
if d.CompactionThresholdRatio <= 0 {
|
||||
d.CompactionThresholdRatio = 0.7
|
||||
}
|
||||
if d.CriticSoftTimeout <= 0 {
|
||||
d.CriticSoftTimeout = 90 * time.Second
|
||||
if d.CriticAbsoluteMax <= 0 {
|
||||
d.CriticAbsoluteMax = 24 * time.Hour
|
||||
}
|
||||
return d
|
||||
}
|
||||
@@ -113,13 +124,26 @@ type Result struct {
|
||||
func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocation, input string) (res Result) {
|
||||
started := time.Now()
|
||||
res = Result{RunID: inv.RunID}
|
||||
// ckpt is the per-run durable checkpointer (resolved below; nil = non-durable).
|
||||
// checkpointCause yields the run context's cancellation cause once the run
|
||||
// context exists; nil before then (an early build-error return).
|
||||
var ckpt Checkpointer
|
||||
var checkpointCause func() error
|
||||
// Enforce the no-panic contract: a panic anywhere in the run (incl. a host
|
||||
// Critic/Audit/Palette callback on the main goroutine) becomes Result.Err
|
||||
// rather than unwinding into the caller.
|
||||
// rather than unwinding into the caller. This defer ALSO finalizes the
|
||||
// checkpoint on EVERY exit path — panic, an early build-error return (before
|
||||
// the run loop), or normal completion — so a recovered run's durable record is
|
||||
// never left dangling (which would loop boot-recovery on a persistent error).
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
res.Err = fmt.Errorf("run.Executor: recovered panic: %v", r)
|
||||
}
|
||||
var cause error
|
||||
if checkpointCause != nil {
|
||||
cause = checkpointCause()
|
||||
}
|
||||
finalizeCheckpoint(ctx, ckpt, res.Err, cause)
|
||||
}()
|
||||
|
||||
tier := ra.ModelTier
|
||||
@@ -165,7 +189,9 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
Name: ra.Name,
|
||||
CallerID: inv.CallerID,
|
||||
ChannelID: inv.ChannelID,
|
||||
GuildID: inv.GuildID,
|
||||
ParentRunID: inv.ParentRunID,
|
||||
ModelTier: tier,
|
||||
Inputs: inv.SkillInputs,
|
||||
StartedAt: started,
|
||||
MaxIterations: maxIter,
|
||||
@@ -180,6 +206,25 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
inv.RunState = stateAcc
|
||||
}
|
||||
|
||||
// Durable recovery (optional): a recovered run carries a ResumeState (prior
|
||||
// transcript / completed phases) + an existing Checkpointer in ctx so it
|
||||
// continues on the SAME durable record; a fresh run mints a per-run
|
||||
// Checkpointer via the factory (which decides durability — nil = non-durable).
|
||||
// nil-safe throughout.
|
||||
resume := resumeStateFromContext(ctx)
|
||||
ckpt = existingCheckpointerFromContext(ctx)
|
||||
if ckpt == nil && e.cfg.Ports.Checkpointer != nil {
|
||||
c, cerr := e.cfg.Ports.Checkpointer.Begin(ctx, info)
|
||||
if cerr != nil {
|
||||
// Degrade to non-durable (the documented contract) but log it — a
|
||||
// failing checkpoint store must not fail the run, yet shouldn't be silent.
|
||||
slog.Warn("run: checkpointer Begin failed; running non-durable",
|
||||
"run_id", inv.RunID, "error", cerr)
|
||||
} else {
|
||||
ckpt = c
|
||||
}
|
||||
}
|
||||
|
||||
// Steer mailbox: lets session tools (via inv.AttachImages) feed multimodal
|
||||
// messages into the running conversation before its next step. Created BEFORE
|
||||
// the toolbox build so any tool's handler captures the live AttachImages seam.
|
||||
@@ -230,18 +275,40 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
postRun = st.PostRun
|
||||
}
|
||||
|
||||
// Run context: bound by MaxRuntime, detached from the caller's deadline so a
|
||||
// lane/queue wait doesn't eat the run budget (mort's V10 lesson). Caller
|
||||
// cancellation still propagates via MergeCancellation. Created BEFORE the
|
||||
// step observer so the observer forwards the merged run context (not a
|
||||
// possibly-cancelled caller ctx) to OnStep consumers.
|
||||
// MaxRuntime stays a WithTimeout so its DeadlineExceeded propagates through the
|
||||
// child chain (→ "timeout"), preserving the run's-own-timeout vs caller-cancel
|
||||
// distinction. A NESTED cause-carrying layer lets a critic kill surface as a
|
||||
// distinct "killed" without disturbing that: only an ErrCriticKill cause is
|
||||
// consulted in statusFor; a generic run error or a caller cancel is classified
|
||||
// by the run error itself.
|
||||
timeoutCtx, cancelTimeout := context.WithTimeout(context.WithoutCancel(ctx), maxRuntime)
|
||||
// Run context: detached from the caller's deadline so a lane/queue wait doesn't
|
||||
// eat the run budget (mort's V10 lesson). Caller cancellation still propagates
|
||||
// via MergeCancellation. Created BEFORE the step observer so the observer
|
||||
// forwards the merged run context (not a possibly-cancelled caller ctx) to
|
||||
// OnStep consumers.
|
||||
//
|
||||
// Two-tier timeout: who owns the hard deadline depends on the critic.
|
||||
// - NO critic (the default): MaxRuntime is a literal WithTimeout. Its
|
||||
// DeadlineExceeded propagates through the child chain (→ "timeout"),
|
||||
// preserving the run's-own-timeout vs caller-cancel distinction.
|
||||
// - critic OWNS the deadline (Ports.Critic set + ra.Critic.Enabled):
|
||||
// MaxRuntime becomes the SOFT trigger (passed to startCritic), and the
|
||||
// critic's extendable backstop — watched in startCritic, which cancels via
|
||||
// cancelCause — is the real deadline. A slow-but-progressing run is given
|
||||
// room up to that backstop; only a stalled one is killed. The base context
|
||||
// gets a WithTimeout at CriticAbsoluteMax (default 24h) purely as a RUNAWAY
|
||||
// guard for a critic that never advances its deadline: it is set FAR beyond
|
||||
// any realistic backstop (the host clamps its own backstop to a much smaller
|
||||
// absolute max, e.g. a 6h host convar), so it does NOT pre-empt a healthy
|
||||
// supervised run. If the host critic fails to ARM (nil handle), the run is
|
||||
// unsupervised and we tighten the cap back down to MaxRuntime below.
|
||||
// A NESTED cause-carrying layer (cancelCause) lets a critic kill surface as a
|
||||
// distinct "killed": only an ErrCriticKill cause is consulted in statusFor; a
|
||||
// generic run error, a backstop expiry, or a caller cancel is classified by the
|
||||
// run error itself.
|
||||
criticOwns := e.criticOwnsDeadline(ra)
|
||||
hardCap := maxRuntime
|
||||
if criticOwns {
|
||||
// Runaway guard only — the critic's own (extendable) deadline-watch is the
|
||||
// normal cap. max() keeps it from being shorter than the nominal budget if an
|
||||
// operator sets MaxRuntime above the runaway ceiling (a degenerate config).
|
||||
hardCap = max(e.cfg.Defaults.CriticAbsoluteMax, maxRuntime)
|
||||
}
|
||||
timeoutCtx, cancelTimeout := context.WithTimeout(context.WithoutCancel(ctx), hardCap)
|
||||
defer cancelTimeout()
|
||||
runCtx, cancelCause := context.WithCancelCause(timeoutCtx)
|
||||
defer cancelCause(nil)
|
||||
@@ -249,11 +316,29 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
defer mergeCancel()
|
||||
|
||||
// Critic (optional): monitors the run for a stall, can nudge/extend/kill via
|
||||
// its host Escalator. Its hard deadline is bound to runCtx (cancel on pass).
|
||||
// nil-safe: no-op when no critic is configured or the agent doesn't enable it.
|
||||
critic, stopCritic := e.startCritic(runCtx, cancelCause, ra, info)
|
||||
// its host Escalator. When it owns the deadline, MaxRuntime is its soft trigger
|
||||
// (so a slow-but-progressing run survives past it); its extendable backstop is
|
||||
// bound to runCtx (cancel on pass). nil-safe: no-op when no critic is configured
|
||||
// or the agent doesn't enable it.
|
||||
critic, stopCritic := e.startCritic(runCtx, cancelCause, ra, info, maxRuntime)
|
||||
defer stopCritic()
|
||||
|
||||
// Unsupervised-run failsafe: the agent enabled the critic (so the base context
|
||||
// got the generous runaway ceiling instead of MaxRuntime), but the host Monitor
|
||||
// returned no handle — there is no deadline-watch. Without this the run would be
|
||||
// bounded only by the 24h ceiling. Tighten it back to the nominal MaxRuntime so
|
||||
// an unsupervised run can't hold its slot far past budget. mort's adapter always
|
||||
// arms when the flag is set, so this is pure defence in depth.
|
||||
if criticOwns && critic == nil {
|
||||
var cancelUnsupervised context.CancelFunc
|
||||
runCtx, cancelUnsupervised = context.WithTimeout(runCtx, maxRuntime)
|
||||
defer cancelUnsupervised()
|
||||
}
|
||||
// The finalize defer (top of Run) now has a run context to read the
|
||||
// cancellation cause from (shutdown vs critic-kill vs deadline vs cancel). Set
|
||||
// AFTER the unsupervised-failsafe re-wrap so it reads the context the loop runs on.
|
||||
checkpointCause = func() error { return context.Cause(runCtx) }
|
||||
|
||||
// Step instrumentation: accumulate Result.Steps + fire inv.OnStep, feed the
|
||||
// audit recorder, and keep the live iteration counter fresh. majordomo's
|
||||
// step observer hands us each completed iteration; we zip the model's tool
|
||||
@@ -289,11 +374,11 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
}
|
||||
|
||||
// Shared agent options used by BOTH the single-loop path and every phase: the
|
||||
// tool-error guards, the step observer, and optional compaction. The toolbox +
|
||||
// step ceiling are NOT shared (they vary per phase), so they're added per path.
|
||||
// tool-error guards and optional compaction. The toolbox, step ceiling, AND
|
||||
// step observer are added per path (the observer is wrapped for checkpointing,
|
||||
// which differs single-loop vs per-phase).
|
||||
sharedOpts := []agent.Option{
|
||||
agent.WithToolErrorLimits(e.cfg.Defaults.MaxConsecutiveToolErrors, e.cfg.Defaults.MaxSameToolCallRepeats),
|
||||
agent.WithStepObserver(stepObserver),
|
||||
}
|
||||
if e.cfg.Compactor != nil && e.cfg.ContextTokens != nil {
|
||||
if threshold := e.compactionThreshold(tier); threshold > 0 {
|
||||
@@ -324,24 +409,60 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
// the critic's nudges before each step.
|
||||
steer := func() []llm.Message { return append(mailbox.drain(), critic.drainSteer()...) }
|
||||
|
||||
resuming := resume != nil && len(resume.History) > 0
|
||||
|
||||
var runRes *agent.Result
|
||||
var runErr error
|
||||
if len(ra.Phases) == 0 {
|
||||
// Single-loop run: the agent's base prompt + full toolbox, with the
|
||||
// critic's DYNAMIC step ceiling (WithMaxStepsFunc, so it can raise a
|
||||
// healthy-but-long run's budget mid-flight; falls back to maxIter).
|
||||
//
|
||||
// Checkpointing: wrap the step observer to accumulate the running transcript
|
||||
// and Save it each step. Save is called every step; THROTTLING is the
|
||||
// Checkpointer's responsibility (the battery + mort's durable-job adapter
|
||||
// both throttle + size-cap), so the kernel doesn't gate the hot path. The
|
||||
// accumulated transcript is the pre-compaction one (the observer sees raw
|
||||
// step responses, not the loop's compacted history) — a host that caps size
|
||||
// bounds it. A recovered run seeds the saved transcript and continues.
|
||||
obs := stepObserver
|
||||
if ckpt != nil {
|
||||
var acc []llm.Message
|
||||
if resuming {
|
||||
acc = append([]llm.Message(nil), resume.History...)
|
||||
} else {
|
||||
acc = []llm.Message{multimodalUserMessage(input, inv.Images)}
|
||||
}
|
||||
obs = func(s agent.Step) {
|
||||
stepObserver(s)
|
||||
if s.Response != nil {
|
||||
acc = append(acc, s.Response.Message())
|
||||
}
|
||||
if len(s.Results) > 0 {
|
||||
acc = append(acc, llm.ToolResultsMessage(s.Results...))
|
||||
}
|
||||
_ = ckpt.Save(runCtx, RunCheckpointState{Messages: acc, Iteration: s.Index + 1})
|
||||
}
|
||||
}
|
||||
opts := append([]agent.Option{
|
||||
agent.WithToolbox(toolbox),
|
||||
critic.maxStepsOption(maxIter),
|
||||
agent.WithStepObserver(obs),
|
||||
}, sharedOpts...)
|
||||
ag := agent.New(model, e.systemPrompt(ra), opts...)
|
||||
if resuming {
|
||||
// Resume: seed the saved transcript and continue (no new input — the
|
||||
// completed tool calls in the transcript are NOT re-run).
|
||||
runRes, runErr = ag.Run(runCtx, "", agent.WithSteer(steer), agent.WithHistory(resume.History))
|
||||
} else {
|
||||
runRes, runErr = runAgent(runCtx, ag, input, inv.Images, agent.WithSteer(steer))
|
||||
}
|
||||
} else {
|
||||
// Multi-phase pipeline: each phase runs its own prompt/tier/tools/step-cap
|
||||
// sequentially, threading outputs through {{.<PhaseName>}} templates. Reuses
|
||||
// the shared opts so audit/steps/critic-steer accumulate across every phase.
|
||||
// (Per-phase step caps are fixed — the critic's dynamic ceiling is not
|
||||
// propagated to phases — but its steer + hard deadline still apply.)
|
||||
// sequentially, threading outputs through {{.<PhaseName>}} templates. The
|
||||
// shared step observer (audit/steps/critic) is wired per phase by the phase
|
||||
// runner; checkpointing is phase-boundary granular (completed phases are
|
||||
// recorded so a resumed run skips them).
|
||||
runRes, runErr = e.runPhases(runCtx, ra, phaseDeps{
|
||||
baseModel: model,
|
||||
baseToolbox: toolbox,
|
||||
@@ -350,9 +471,14 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
stepObserver: stepObserver,
|
||||
steer: steer,
|
||||
rec: rec,
|
||||
checkpointer: ckpt,
|
||||
resume: resume,
|
||||
}, input, inv.Images)
|
||||
}
|
||||
|
||||
// Durable-recovery finalize (Complete/Fail/leave-running) happens in the
|
||||
// top-of-Run defer so it covers panics + early build-error returns too.
|
||||
|
||||
status := statusFor(runCtx, runErr)
|
||||
if runRes != nil {
|
||||
res.Output = runRes.Output
|
||||
|
||||
+46
-5
@@ -53,9 +53,10 @@ import (
|
||||
|
||||
// phaseDeps carries the per-run state the phase runner shares with Run: the base
|
||||
// model, the full decorated toolbox (filtered per phase), the base step cap, the
|
||||
// shared agent options (tool-error limits + step observer + compactor), the
|
||||
// shared step observer (also fed by IsRunFunc bare calls), the critic/session
|
||||
// steer, and the audit recorder (phase events).
|
||||
// shared agent options (tool-error limits + compactor — the step observer is
|
||||
// added per phase, NOT in sharedOpts, so checkpointing can vary per path), the
|
||||
// shared step observer (wired into each phase's loop AND invoked for IsRunFunc
|
||||
// bare calls), the critic/session steer, and the audit recorder (phase events).
|
||||
type phaseDeps struct {
|
||||
baseModel llm.Model
|
||||
baseToolbox *llm.Toolbox
|
||||
@@ -64,6 +65,13 @@ type phaseDeps struct {
|
||||
stepObserver func(agent.Step)
|
||||
steer func() []llm.Message
|
||||
rec RunRecorder
|
||||
// checkpointer records phase-boundary progress (completed phases) for durable
|
||||
// recovery; nil = non-durable. resume carries a recovered run's completed
|
||||
// phases so they are skipped on re-run. Phase recovery is boundary-granular:
|
||||
// the interrupted (active) phase re-runs from its start (its mid-phase
|
||||
// transcript is NOT resumed — only the single-loop path resumes mid-loop).
|
||||
checkpointer Checkpointer
|
||||
resume *ResumeState
|
||||
}
|
||||
|
||||
// runPhases executes ra.Phases sequentially and returns a synthetic agent.Result
|
||||
@@ -73,10 +81,28 @@ type phaseDeps struct {
|
||||
// deadline/critic-kill — returns the error.
|
||||
func (e *Executor) runPhases(runCtx context.Context, ra RunnableAgent, deps phaseDeps, query string, images []llm.ImagePart) (*agent.Result, error) {
|
||||
outputs := make(map[string]string, len(ra.Phases))
|
||||
var completed []PhaseOutput
|
||||
var lastResult *agent.Result
|
||||
var lastOutput string
|
||||
var totalUsage llm.Usage
|
||||
|
||||
// resumeSkip is the set of phases already finished on a RECOVERED run — kept
|
||||
// SEPARATE from the live `outputs` map (which fills as phases run this time) so
|
||||
// the skip guard only skips RESUME-completed phases, never a fresh run's own
|
||||
// phases. (Reusing `outputs` would make a second phase with a duplicate name
|
||||
// skip itself.) Pre-populate outputs + completed so a resumed run threads the
|
||||
// saved outputs into later phases. The interrupted (active) phase is NOT
|
||||
// pre-populated, so it re-runs from its start (boundary-granular recovery).
|
||||
resumeSkip := map[string]bool{}
|
||||
if deps.resume != nil {
|
||||
for _, pc := range deps.resume.CompletedPhases {
|
||||
outputs[pc.Name] = pc.Output
|
||||
resumeSkip[pc.Name] = true
|
||||
completed = append(completed, pc)
|
||||
lastOutput = pc.Output
|
||||
}
|
||||
}
|
||||
|
||||
// finish stamps the aggregated usage + final output onto the synthetic result.
|
||||
finish := func(err error) (*agent.Result, error) {
|
||||
if lastResult == nil {
|
||||
@@ -90,6 +116,10 @@ func (e *Executor) runPhases(runCtx context.Context, ra RunnableAgent, deps phas
|
||||
}
|
||||
|
||||
for i, phase := range ra.Phases {
|
||||
// Skip phases already completed on a resumed run.
|
||||
if resumeSkip[phase.Name] {
|
||||
continue
|
||||
}
|
||||
// A killed/timed-out/cancelled run must not start its next phase.
|
||||
if err := runCtx.Err(); err != nil {
|
||||
return finish(err)
|
||||
@@ -151,6 +181,15 @@ func (e *Executor) runPhases(runCtx context.Context, ra RunnableAgent, deps phas
|
||||
|
||||
outputs[phase.Name] = output
|
||||
lastOutput = output
|
||||
// Checkpoint the phase boundary: this phase is done, so a resumed run skips
|
||||
// it and continues from the next. (Copy the slice — the checkpointer may
|
||||
// hold/serialize it asynchronously.)
|
||||
completed = append(completed, PhaseOutput{Name: phase.Name, Output: output})
|
||||
if deps.checkpointer != nil {
|
||||
_ = deps.checkpointer.Save(runCtx, RunCheckpointState{
|
||||
CompletedPhases: append([]PhaseOutput(nil), completed...),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return finish(nil)
|
||||
@@ -192,11 +231,13 @@ func (e *Executor) runOnePhase(runCtx context.Context, ra RunnableAgent, deps ph
|
||||
maxIter = deps.baseMaxIter
|
||||
}
|
||||
// Per-phase opts: a fixed step ceiling for this phase (the critic's dynamic
|
||||
// ceiling is intentionally not propagated to phases) + the phase toolbox, on
|
||||
// top of the shared opts (tool-error limits, step observer, compactor).
|
||||
// ceiling is intentionally not propagated to phases) + the phase toolbox + the
|
||||
// shared step observer (audit/steps/critic), on top of the shared opts
|
||||
// (tool-error limits, compactor).
|
||||
opts := append([]agent.Option{
|
||||
agent.WithToolbox(toolbox),
|
||||
agent.WithMaxSteps(maxIter),
|
||||
agent.WithStepObserver(deps.stepObserver),
|
||||
}, deps.sharedOpts...)
|
||||
ag := agent.New(model, system, opts...)
|
||||
|
||||
|
||||
@@ -130,6 +130,26 @@ func TestPhases_OptionalDoesNotSwallowCancellation(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPhases_DuplicateNamesBothRun: a fresh (non-resume) run with two phases
|
||||
// sharing a name must run BOTH — the resume-skip guard keys off a separate
|
||||
// resume set, not the live outputs map (which fills as phases run), so a phase
|
||||
// never skips a same-named sibling on a fresh run.
|
||||
func TestPhases_DuplicateNamesBothRun(t *testing.T) {
|
||||
models, fp := phaseProvider(t, fake.Reply("first"), fake.Reply("second"))
|
||||
ex := New(Config{Registry: tool.NewRegistry(), Models: models})
|
||||
ra := RunnableAgent{
|
||||
Name: "p", ModelTier: "test-model",
|
||||
Phases: []Phase{{Name: "x", SystemPrompt: "P1"}, {Name: "x", SystemPrompt: "P2"}},
|
||||
}
|
||||
res := ex.Run(context.Background(), ra, tool.Invocation{RunID: "r"}, "Q")
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if n := len(fp.Calls()); n != 2 {
|
||||
t.Fatalf("both same-named phases must run on a fresh run; got %d model calls", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPhases_HardErrorAborts: a NON-optional phase that hits a hard error (not a
|
||||
// budget/step exhaustion) aborts the pipeline; later phases do not run.
|
||||
func TestPhases_HardErrorAborts(t *testing.T) {
|
||||
|
||||
+31
-5
@@ -33,9 +33,10 @@ type Ports struct {
|
||||
Budget Budget
|
||||
// Critic optionally monitors a long run for hangs/runaways. nil = none.
|
||||
Critic Critic
|
||||
// Checkpointer persists resumable progress for durable recovery. nil = no
|
||||
// checkpointing (a run interrupted by shutdown is simply lost).
|
||||
Checkpointer Checkpointer
|
||||
// Checkpointer mints a per-run Checkpointer for durable recovery (it decides
|
||||
// per run whether the run is durable). nil = no checkpointing (a run
|
||||
// interrupted by shutdown is simply lost).
|
||||
Checkpointer CheckpointerFactory
|
||||
// Palette resolves SkillPalette / SubAgentPalette entries into delegation
|
||||
// tools (skill__<name> / agent__<name>). nil = those entries are inert.
|
||||
Palette PaletteSource
|
||||
@@ -66,7 +67,9 @@ type RunInfo struct {
|
||||
Name string
|
||||
CallerID string
|
||||
ChannelID string
|
||||
GuildID string // the originating guild/server id (empty for DMs/triggers)
|
||||
ParentRunID string
|
||||
ModelTier string // the run's resolved base tier (for checkpoint re-dispatch)
|
||||
Inputs map[string]any
|
||||
StartedAt time.Time
|
||||
// MaxIterations is the run's base tool-dispatch step ceiling, so a critic can
|
||||
@@ -172,6 +175,16 @@ type CriticHandle interface {
|
||||
|
||||
// --- Checkpointer ---
|
||||
|
||||
// CheckpointerFactory decides, per run, whether the run is durable and (if so)
|
||||
// mints the per-run Checkpointer that records its progress. It returns (nil, nil)
|
||||
// for a non-durable run (the common short-run case — no checkpointing overhead).
|
||||
// A storage error should be logged and degraded to (nil, nil) so a failing
|
||||
// checkpoint store never fails the run. Mirrors mort's
|
||||
// agentexec.CheckpointerFactory.
|
||||
type CheckpointerFactory interface {
|
||||
Begin(ctx context.Context, info RunInfo) (Checkpointer, error)
|
||||
}
|
||||
|
||||
// Checkpointer persists a run's resumable progress for durable recovery.
|
||||
// Mirrors mort's agentexec.RunCheckpointer.
|
||||
type Checkpointer interface {
|
||||
@@ -184,11 +197,24 @@ type Checkpointer interface {
|
||||
Fail(ctx context.Context, err error) error
|
||||
}
|
||||
|
||||
// RunCheckpointState is the resumable snapshot a Checkpointer persists. Kept
|
||||
// minimal here; the executor extends what it records during the merge.
|
||||
// RunCheckpointState is the resumable snapshot a Checkpointer persists.
|
||||
type RunCheckpointState struct {
|
||||
// Messages is the running transcript of a SINGLE-LOOP run (grows each step;
|
||||
// resumed via WithHistory). nil for multi-phase runs — phase recovery is
|
||||
// boundary-granular (see CompletedPhases), not mid-phase transcript.
|
||||
Messages []llm.Message
|
||||
Iteration int
|
||||
// CompletedPhases is set only for multi-phase runs: the outputs of phases
|
||||
// already finished, in phase order, so a resumed run skips them and re-runs
|
||||
// the interrupted phase from its start. nil for single-loop runs.
|
||||
CompletedPhases []PhaseOutput
|
||||
}
|
||||
|
||||
// PhaseOutput is one completed pipeline phase's name and output text, recorded in
|
||||
// a checkpoint so a resumed multi-phase run can skip already-finished phases.
|
||||
type PhaseOutput struct {
|
||||
Name string
|
||||
Output string
|
||||
}
|
||||
|
||||
// --- PaletteSource ---
|
||||
|
||||
Reference in New Issue
Block a user