fix(run): address gadfly review of the critic-deadline PR
executus CI / test (pull_request) Successful in 1m45s
executus CI / test (pull_request) Successful in 1m45s
All 11 findings were real (3 clusters): - Failsafe ceiling could pre-empt the critic's backstop (e9c9483f, 9109317b, d5a9bf0d, 76ad171e): CriticAbsoluteMax was 6h, but the host's backstop (MaxRuntime × multiplier, or its own absolute max) can reach 6h+, so the ceiling fired first and reintroduced a premature hard cap. Now CriticAbsoluteMax is a 24h RUNAWAY guard set far beyond any realistic backstop (the host clamps its own backstop to a much smaller absolute max, e.g. mort's 6h convar), so it never pre-empts a healthy supervised run. Comments corrected. - nil Monitor handle lost the MaxRuntime cap (df016a6f, 9dd42827): a critic-enabled run whose host Monitor returned no handle had no deadline-watch and was bounded only by the generous ceiling. Added an unsupervised-run failsafe that re-wraps runCtx to the nominal MaxRuntime when the critic is enabled but didn't arm. New test TestCriticOwnsDeadline_NilHandleFallsBackToMaxRuntime. - CriticSoftTimeout vestigial / dead fallback (f7764919, 9805bebe, 6864086f, b2b11721): the soft trigger is now always the resolved MaxRuntime (> 0), so the CriticSoftTimeout field + its startCritic fallback were unreachable. Removed the field entirely; the remaining 90s floor is documented as defensive-only. - DRY (f30ce827): extracted e.criticOwnsDeadline(ra), now the single predicate used by both Run and startCritic so they can't drift. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Jo75sqmeVPgFUWZQBn179X
This commit is contained in:
+13
-6
@@ -22,6 +22,14 @@ type criticBinding struct {
|
|||||||
h CriticHandle
|
h CriticHandle
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// criticOwnsDeadline reports whether a critic is configured AND this run enables
|
||||||
|
// it — the single predicate that decides the two-tier-timeout path. Used by BOTH
|
||||||
|
// Run (to choose the generous runaway ceiling over the literal MaxRuntime cap) and
|
||||||
|
// startCritic (the arm/no-op gate), so the two can never drift.
|
||||||
|
func (e *Executor) criticOwnsDeadline(ra RunnableAgent) bool {
|
||||||
|
return e.cfg.Ports.Critic != nil && ra.Critic.Enabled
|
||||||
|
}
|
||||||
|
|
||||||
// startCritic begins critic monitoring for this run when one is configured and
|
// startCritic begins critic monitoring for this run when one is configured and
|
||||||
// the agent enables it. It launches a goroutine that cancels runCtx (via
|
// the agent enables it. It launches a goroutine that cancels runCtx (via
|
||||||
// cancelCause) the moment the critic's hard deadline passes — the critic may
|
// cancelCause) the moment the critic's hard deadline passes — the critic may
|
||||||
@@ -35,18 +43,17 @@ type criticBinding struct {
|
|||||||
// softTrigger is the run's resolved MaxRuntime: for a critic-owned run MaxRuntime
|
// softTrigger is the run's resolved MaxRuntime: for a critic-owned run MaxRuntime
|
||||||
// is the soft wake (mort's two-tier semantics — the critic first reviews once the
|
// is the soft wake (mort's two-tier semantics — the critic first reviews once the
|
||||||
// run exceeds its nominal budget, and its backstop = softTrigger × multiplier).
|
// run exceeds its nominal budget, and its backstop = softTrigger × multiplier).
|
||||||
// It falls back to the configured CriticSoftTimeout when the run set no MaxRuntime.
|
// The caller (Run) always passes the resolved MaxRuntime, which withFallbacks
|
||||||
|
// guarantees is > 0; the 90s floor below is purely a defensive guard for a
|
||||||
|
// hypothetical caller that passes a non-positive value.
|
||||||
func (e *Executor) startCritic(runCtx context.Context, cancelCause context.CancelCauseFunc, ra RunnableAgent, info RunInfo, softTrigger time.Duration) (*criticBinding, func()) {
|
func (e *Executor) startCritic(runCtx context.Context, cancelCause context.CancelCauseFunc, ra RunnableAgent, info RunInfo, softTrigger time.Duration) (*criticBinding, func()) {
|
||||||
noop := func() {}
|
noop := func() {}
|
||||||
if e.cfg.Ports.Critic == nil || !ra.Critic.Enabled {
|
if !e.criticOwnsDeadline(ra) {
|
||||||
return nil, noop
|
return nil, noop
|
||||||
}
|
}
|
||||||
soft := softTrigger
|
soft := softTrigger
|
||||||
if soft <= 0 {
|
if soft <= 0 {
|
||||||
soft = e.cfg.Defaults.CriticSoftTimeout
|
soft = 90 * time.Second // defensive only; the sole caller passes MaxRuntime (>0)
|
||||||
}
|
|
||||||
if soft <= 0 {
|
|
||||||
soft = 90 * time.Second // defensive: withFallbacks normally guarantees >0
|
|
||||||
}
|
}
|
||||||
h := e.cfg.Ports.Critic.Monitor(runCtx, info, soft)
|
h := e.cfg.Ports.Critic.Monitor(runCtx, info, soft)
|
||||||
if h == nil {
|
if h == nil {
|
||||||
|
|||||||
@@ -106,7 +106,7 @@ func (c *capturingCritic) Monitor(_ context.Context, _ run.RunInfo, soft time.Du
|
|||||||
|
|
||||||
// TestCriticSoftTriggerIsMaxRuntime: the soft trigger handed to the host critic is
|
// TestCriticSoftTriggerIsMaxRuntime: the soft trigger handed to the host critic is
|
||||||
// the run's resolved MaxRuntime (mort's two-tier model — the critic first wakes once
|
// the run's resolved MaxRuntime (mort's two-tier model — the critic first wakes once
|
||||||
// the run exceeds its nominal budget), NOT the global Defaults.CriticSoftTimeout.
|
// the run exceeds its nominal budget), not some global/default value.
|
||||||
func TestCriticSoftTriggerIsMaxRuntime(t *testing.T) {
|
func TestCriticSoftTriggerIsMaxRuntime(t *testing.T) {
|
||||||
fp := fake.New("fake")
|
fp := fake.New("fake")
|
||||||
fp.Enqueue("m", fake.Reply("done"))
|
fp.Enqueue("m", fake.Reply("done"))
|
||||||
@@ -116,7 +116,6 @@ func TestCriticSoftTriggerIsMaxRuntime(t *testing.T) {
|
|||||||
Registry: tool.NewRegistry(),
|
Registry: tool.NewRegistry(),
|
||||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||||
Ports: run.Ports{Critic: cc},
|
Ports: run.Ports{Critic: cc},
|
||||||
Defaults: run.Defaults{CriticSoftTimeout: 90 * time.Second}, // distinct from MaxRuntime below
|
|
||||||
})
|
})
|
||||||
const wantSoft = 7 * time.Minute
|
const wantSoft = 7 * time.Minute
|
||||||
ex.Run(context.Background(),
|
ex.Run(context.Background(),
|
||||||
@@ -126,6 +125,29 @@ func TestCriticSoftTriggerIsMaxRuntime(t *testing.T) {
|
|||||||
got := cc.soft
|
got := cc.soft
|
||||||
cc.mu.Unlock()
|
cc.mu.Unlock()
|
||||||
if got != wantSoft {
|
if got != wantSoft {
|
||||||
t.Errorf("soft trigger = %v, want the agent's MaxRuntime %v (not Defaults.CriticSoftTimeout)", got, wantSoft)
|
t.Errorf("soft trigger = %v, want the agent's MaxRuntime %v", got, wantSoft)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCriticOwnsDeadline_NilHandleFallsBackToMaxRuntime: the agent enables the
|
||||||
|
// critic but the host Monitor returns NO handle (nil) — there is no deadline-watch,
|
||||||
|
// so the run is unsupervised. It must fall back to the nominal MaxRuntime hard cap
|
||||||
|
// (the slow 200ms tool outlasts the 20ms MaxRuntime → the run errors), NOT run free
|
||||||
|
// up to the generous CriticAbsoluteMax runaway ceiling.
|
||||||
|
func TestCriticOwnsDeadline_NilHandleFallsBackToMaxRuntime(t *testing.T) {
|
||||||
|
m := slowModel()
|
||||||
|
cc := &capturingCritic{} // h is the nil interface → Monitor returns a nil handle
|
||||||
|
ex := run.New(run.Config{
|
||||||
|
Registry: tool.NewRegistry(),
|
||||||
|
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||||
|
Ports: run.Ports{Critic: cc},
|
||||||
|
Defaults: run.Defaults{CriticAbsoluteMax: time.Hour}, // generous ceiling; must NOT be what bounds the run
|
||||||
|
})
|
||||||
|
res := ex.Run(context.Background(),
|
||||||
|
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 5, MaxRuntime: 20 * time.Millisecond,
|
||||||
|
Critic: run.CriticConfig{Enabled: true}},
|
||||||
|
slowToolInvocation("r", 200*time.Millisecond), "go")
|
||||||
|
if res.Err == nil {
|
||||||
|
t.Fatalf("critic-enabled run with a nil Monitor handle must fall back to the MaxRuntime hard cap; got output=%q err=nil", res.Output)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -61,8 +61,8 @@ func TestCriticRaisesStepCeiling(t *testing.T) {
|
|||||||
Registry: tool.NewRegistry(),
|
Registry: tool.NewRegistry(),
|
||||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||||
Ports: run.Ports{Critic: &fakeCritic{h: h}},
|
Ports: run.Ports{Critic: &fakeCritic{h: h}},
|
||||||
// large soft timeout so the deadline-watch never interferes in the test
|
// The fake handle's Deadline() is zero (no hard deadline), so the
|
||||||
Defaults: run.Defaults{CriticSoftTimeout: time.Hour},
|
// deadline-watch never interferes regardless of the soft trigger.
|
||||||
})
|
})
|
||||||
res := ex.Run(context.Background(),
|
res := ex.Run(context.Background(),
|
||||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 1, Critic: run.CriticConfig{Enabled: true}},
|
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 1, Critic: run.CriticConfig{Enabled: true}},
|
||||||
|
|||||||
+41
-20
@@ -29,13 +29,17 @@ type Defaults struct {
|
|||||||
MaxConsecutiveToolErrors int // loop guard; default 3
|
MaxConsecutiveToolErrors int // loop guard; default 3
|
||||||
MaxSameToolCallRepeats int // retry-storm guard; default 3
|
MaxSameToolCallRepeats int // retry-storm guard; default 3
|
||||||
CompactionThresholdRatio float64 // fraction of model context to compact at; default 0.7
|
CompactionThresholdRatio float64 // fraction of model context to compact at; default 0.7
|
||||||
CriticSoftTimeout time.Duration // idle window before the critic wakes; default 90s
|
// CriticAbsoluteMax is the RUNAWAY ceiling for a critic-OWNED run (Ports.Critic
|
||||||
// CriticAbsoluteMax is the failsafe wall-clock ceiling for a critic-OWNED run
|
// set AND the agent enables it). For such a run MaxRuntime is the SOFT trigger,
|
||||||
// (Ports.Critic set AND the agent enables it). For such a run MaxRuntime is the
|
// not a hard cap, and the critic's own extendable backstop is the normal
|
||||||
// SOFT trigger, not a hard cap, and the critic's extendable backstop is the
|
// deadline. This ceiling exists ONLY to stop a critic that never advances its
|
||||||
// normal deadline — so this ceiling only fires if the critic never acts (a
|
// deadline (a broken host handle) from running forever, so it is deliberately
|
||||||
// broken/nil host handle). Default 6h; never shorter than the run's MaxRuntime.
|
// set FAR beyond any realistic backstop (default 24h): the host clamps its own
|
||||||
// Non-critic runs ignore it (they keep the literal MaxRuntime kill).
|
// backstop to a much smaller absolute max (e.g. mort's agents.critic.
|
||||||
|
// absolute_max_seconds = 6h), so the ceiling never pre-empts a healthy
|
||||||
|
// supervised run. Keep it well above the host's absolute max. Never shorter than
|
||||||
|
// the run's MaxRuntime. Non-critic runs ignore it (they keep the literal
|
||||||
|
// MaxRuntime kill).
|
||||||
CriticAbsoluteMax time.Duration
|
CriticAbsoluteMax time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,11 +62,8 @@ func (d Defaults) withFallbacks() Defaults {
|
|||||||
if d.CompactionThresholdRatio <= 0 {
|
if d.CompactionThresholdRatio <= 0 {
|
||||||
d.CompactionThresholdRatio = 0.7
|
d.CompactionThresholdRatio = 0.7
|
||||||
}
|
}
|
||||||
if d.CriticSoftTimeout <= 0 {
|
|
||||||
d.CriticSoftTimeout = 90 * time.Second
|
|
||||||
}
|
|
||||||
if d.CriticAbsoluteMax <= 0 {
|
if d.CriticAbsoluteMax <= 0 {
|
||||||
d.CriticAbsoluteMax = 6 * time.Hour
|
d.CriticAbsoluteMax = 24 * time.Hour
|
||||||
}
|
}
|
||||||
return d
|
return d
|
||||||
}
|
}
|
||||||
@@ -289,19 +290,26 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
|||||||
// MaxRuntime becomes the SOFT trigger (passed to startCritic), and the
|
// MaxRuntime becomes the SOFT trigger (passed to startCritic), and the
|
||||||
// critic's extendable backstop — watched in startCritic, which cancels via
|
// critic's extendable backstop — watched in startCritic, which cancels via
|
||||||
// cancelCause — is the real deadline. A slow-but-progressing run is given
|
// cancelCause — is the real deadline. A slow-but-progressing run is given
|
||||||
// room up to the backstop; only a stalled one is killed. We still wrap a
|
// room up to that backstop; only a stalled one is killed. The base context
|
||||||
// GENEROUS WithTimeout at CriticAbsoluteMax so a broken/nil critic handle
|
// gets a WithTimeout at CriticAbsoluteMax (default 24h) purely as a RUNAWAY
|
||||||
// can't run unbounded; that ceiling never fires before the critic's backstop.
|
// guard for a critic that never advances its deadline: it is set FAR beyond
|
||||||
|
// any realistic backstop (the host clamps its own backstop to a much smaller
|
||||||
|
// absolute max, e.g. mort's 6h convar), so it does NOT pre-empt a healthy
|
||||||
|
// supervised run. If the host critic fails to ARM (nil handle), the run is
|
||||||
|
// unsupervised and we tighten the cap back down to MaxRuntime below.
|
||||||
// A NESTED cause-carrying layer (cancelCause) lets a critic kill surface as a
|
// A NESTED cause-carrying layer (cancelCause) lets a critic kill surface as a
|
||||||
// distinct "killed": only an ErrCriticKill cause is consulted in statusFor; a
|
// distinct "killed": only an ErrCriticKill cause is consulted in statusFor; a
|
||||||
// generic run error, a backstop expiry, or a caller cancel is classified by the
|
// generic run error, a backstop expiry, or a caller cancel is classified by the
|
||||||
// run error itself.
|
// run error itself.
|
||||||
criticOwnsDeadline := e.cfg.Ports.Critic != nil && ra.Critic.Enabled
|
criticOwns := e.criticOwnsDeadline(ra)
|
||||||
hardCap := maxRuntime
|
hardCap := maxRuntime
|
||||||
if criticOwnsDeadline {
|
if criticOwns {
|
||||||
|
// Runaway guard only — the critic's own (extendable) deadline-watch is the
|
||||||
|
// normal cap. Never shorter than the nominal budget, in case an operator
|
||||||
|
// sets MaxRuntime above the runaway ceiling (a degenerate config).
|
||||||
hardCap = e.cfg.Defaults.CriticAbsoluteMax
|
hardCap = e.cfg.Defaults.CriticAbsoluteMax
|
||||||
if hardCap < maxRuntime {
|
if hardCap < maxRuntime {
|
||||||
hardCap = maxRuntime // the failsafe ceiling is never shorter than the nominal budget
|
hardCap = maxRuntime
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
timeoutCtx, cancelTimeout := context.WithTimeout(context.WithoutCancel(ctx), hardCap)
|
timeoutCtx, cancelTimeout := context.WithTimeout(context.WithoutCancel(ctx), hardCap)
|
||||||
@@ -310,9 +318,6 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
|||||||
defer cancelCause(nil)
|
defer cancelCause(nil)
|
||||||
runCtx, mergeCancel := MergeCancellation(runCtx, ctx)
|
runCtx, mergeCancel := MergeCancellation(runCtx, ctx)
|
||||||
defer mergeCancel()
|
defer mergeCancel()
|
||||||
// The finalize defer (top of Run) now has a run context to read the
|
|
||||||
// cancellation cause from (shutdown vs critic-kill vs deadline vs cancel).
|
|
||||||
checkpointCause = func() error { return context.Cause(runCtx) }
|
|
||||||
|
|
||||||
// Critic (optional): monitors the run for a stall, can nudge/extend/kill via
|
// Critic (optional): monitors the run for a stall, can nudge/extend/kill via
|
||||||
// its host Escalator. When it owns the deadline, MaxRuntime is its soft trigger
|
// its host Escalator. When it owns the deadline, MaxRuntime is its soft trigger
|
||||||
@@ -322,6 +327,22 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
|||||||
critic, stopCritic := e.startCritic(runCtx, cancelCause, ra, info, maxRuntime)
|
critic, stopCritic := e.startCritic(runCtx, cancelCause, ra, info, maxRuntime)
|
||||||
defer stopCritic()
|
defer stopCritic()
|
||||||
|
|
||||||
|
// Unsupervised-run failsafe: the agent enabled the critic (so the base context
|
||||||
|
// got the generous runaway ceiling instead of MaxRuntime), but the host Monitor
|
||||||
|
// returned no handle — there is no deadline-watch. Without this the run would be
|
||||||
|
// bounded only by the 24h ceiling. Tighten it back to the nominal MaxRuntime so
|
||||||
|
// an unsupervised run can't hold its slot far past budget. mort's adapter always
|
||||||
|
// arms when the flag is set, so this is pure defence in depth.
|
||||||
|
if criticOwns && critic == nil {
|
||||||
|
var cancelUnsupervised context.CancelFunc
|
||||||
|
runCtx, cancelUnsupervised = context.WithTimeout(runCtx, maxRuntime)
|
||||||
|
defer cancelUnsupervised()
|
||||||
|
}
|
||||||
|
// The finalize defer (top of Run) now has a run context to read the
|
||||||
|
// cancellation cause from (shutdown vs critic-kill vs deadline vs cancel). Set
|
||||||
|
// AFTER the unsupervised-failsafe re-wrap so it reads the context the loop runs on.
|
||||||
|
checkpointCause = func() error { return context.Cause(runCtx) }
|
||||||
|
|
||||||
// Step instrumentation: accumulate Result.Steps + fire inv.OnStep, feed the
|
// Step instrumentation: accumulate Result.Steps + fire inv.OnStep, feed the
|
||||||
// audit recorder, and keep the live iteration counter fresh. majordomo's
|
// audit recorder, and keep the live iteration counter fresh. majordomo's
|
||||||
// step observer hands us each completed iteration; we zip the model's tool
|
// step observer hands us each completed iteration; we zip the model's tool
|
||||||
|
|||||||
Reference in New Issue
Block a user