P4: critic battery — two-tier timeout watchdog + Escalator seam
The last Tier-2 battery, plugging into run.Ports.Critic (executor call site is a P2 follow-up). Clean split of concerns: - executus owns the deterministic MECHANICS: System.Monitor returns a run.CriticHandle that tracks activity (RecordStep/RecordToolStart), and a watcher goroutine fires once per idle period a run crosses its soft timeout, applies the decision (queue Steer nudges / extend the Deadline / collapse it to now on Kill), and enforces a hard-kill backstop (softTimeout * mul). - the POLICY is the Escalator seam (nudge/extend/kill/escalate). Mort plugs its LLM critic-agent in here; ExtendOnce is the zero-dependency default (extend once, then let the backstop kill a truly hung run). Race-tested: escalate-once-per-idle-period with re-arm on fresh activity, Kill collapses the deadline, ExtendOnce fires once, zero soft-timeout => nil handle. Core imports ZERO from critic. This completes the P4 battery set: audit, budget, persona, skill, checkpoint, schedule, critic — each nil-safe, each with a default, each core-import-clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
package critic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/run"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// escFunc adapts a func to an Escalator.
|
||||
type escFunc func(context.Context, run.RunInfo, Progress) Decision
|
||||
|
||||
func (f escFunc) OnSoftTimeout(ctx context.Context, i run.RunInfo, p Progress) Decision {
|
||||
return f(ctx, i, p)
|
||||
}
|
||||
|
||||
func TestMonitorEscalatesOncePerIdlePeriodAndExtends(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var calls int
|
||||
esc := escFunc(func(_ context.Context, _ run.RunInfo, p Progress) Decision {
|
||||
mu.Lock()
|
||||
calls++
|
||||
mu.Unlock()
|
||||
return Decision{ExtendBy: 50 * time.Millisecond, Nudge: []llm.Message{{Role: llm.RoleUser}}}
|
||||
})
|
||||
s := New(esc, 3)
|
||||
s.checkInterval = 5 * time.Millisecond
|
||||
h := s.Monitor(context.Background(), run.RunInfo{RunID: "r"}, 20*time.Millisecond)
|
||||
defer h.Stop()
|
||||
|
||||
d0 := h.Deadline()
|
||||
time.Sleep(60 * time.Millisecond) // cross the soft timeout with no activity
|
||||
mu.Lock()
|
||||
c := calls
|
||||
mu.Unlock()
|
||||
if c < 1 {
|
||||
t.Fatalf("expected at least one escalation, got %d", c)
|
||||
}
|
||||
// Nudge was queued and is drained once.
|
||||
if msgs := h.Steer(); len(msgs) == 0 {
|
||||
t.Error("expected a queued steer nudge")
|
||||
}
|
||||
if msgs := h.Steer(); len(msgs) != 0 {
|
||||
t.Error("steer should drain (be empty on second read)")
|
||||
}
|
||||
// Deadline was extended.
|
||||
if !h.Deadline().After(d0) {
|
||||
t.Error("deadline should have been extended past the original")
|
||||
}
|
||||
// A fresh step re-arms; another idle period escalates again.
|
||||
h.RecordStep(1)
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
mu.Lock()
|
||||
c2 := calls
|
||||
mu.Unlock()
|
||||
if c2 <= c {
|
||||
t.Errorf("a re-armed idle period should escalate again (%d -> %d)", c, c2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKillCollapsesDeadline(t *testing.T) {
|
||||
esc := escFunc(func(context.Context, run.RunInfo, Progress) Decision {
|
||||
return Decision{Kill: true, KillReason: "hung"}
|
||||
})
|
||||
s := New(esc, 10) // big backstop so only Kill collapses it
|
||||
s.checkInterval = 5 * time.Millisecond
|
||||
h := s.Monitor(context.Background(), run.RunInfo{RunID: "r"}, 20*time.Millisecond)
|
||||
defer h.Stop()
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
if h.Deadline().After(time.Now().Add(time.Second)) {
|
||||
t.Error("Kill should collapse the deadline to ~now")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtendOnceOnlyFiresOnce(t *testing.T) {
|
||||
e := &ExtendOnce{By: time.Minute}
|
||||
d1 := e.OnSoftTimeout(context.Background(), run.RunInfo{}, Progress{})
|
||||
d2 := e.OnSoftTimeout(context.Background(), run.RunInfo{}, Progress{})
|
||||
if d1.ExtendBy != time.Minute {
|
||||
t.Errorf("first decision should extend, got %+v", d1)
|
||||
}
|
||||
if d2.ExtendBy != 0 || d2.Kill {
|
||||
t.Errorf("second decision should be a no-op, got %+v", d2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZeroSoftTimeoutNotMonitored(t *testing.T) {
|
||||
s := New(nil, 3)
|
||||
if h := s.Monitor(context.Background(), run.RunInfo{}, 0); h != nil {
|
||||
t.Error("zero soft timeout should return a nil handle (not monitored)")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user