steve
eea84e6e2c
executus CI / test (pull_request) Successful in 1m39s
critic (all 3 models — HIGH): - ExtendOnce was a single global one-shot shared across every run a System monitors, so only the FIRST run to stall got its extension and all others were killed by the backstop. Key the fired-state per run (RunInfo.RunID). - Kill is now sticky: a `killed` flag short-circuits later ticks so a wavering Escalator returning ExtendBy after a Kill can't un-collapse the deadline; a Kill paired with Nudge/ExtendBy ignores the latter. - watch() recovers panics from a misbehaving Escalator (logs; the run falls back to its existing deadline) instead of silently killing the watch goroutine. checkpoint (deepseek — HIGH): handle.Save advanced the throttle clock BEFORE the store write, so a failed save was silently throttled away (caller believes it persisted). Advance lastSave only after a successful persist. schedule (all 3): compute Next BEFORE Run — a permanently-unparseable cron now skips the job entirely instead of re-running it every tick forever; nil required callbacks return a validate() error instead of a first-tick nil panic; Loop recovers tick panics; the Mark-failure => possible-re-run trade-off is documented (Run must be idempotent). + tests for each. Triaged-but-kept: critic backstopMul<=1 floor (it's a total-runtime multiple, so a floor >1 is intentional, not the reported footgun); checkpoint Load (nil,nil) on miss (documented convention). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
101 lines
3.1 KiB
Go
101 lines
3.1 KiB
Go
package critic
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.stevedudenhoeffer.com/steve/executus/run"
|
|
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
|
)
|
|
|
|
// escFunc adapts a func to an Escalator.
|
|
type escFunc func(context.Context, run.RunInfo, Progress) Decision
|
|
|
|
func (f escFunc) OnSoftTimeout(ctx context.Context, i run.RunInfo, p Progress) Decision {
|
|
return f(ctx, i, p)
|
|
}
|
|
|
|
func TestMonitorEscalatesOncePerIdlePeriodAndExtends(t *testing.T) {
|
|
var mu sync.Mutex
|
|
var calls int
|
|
esc := escFunc(func(_ context.Context, _ run.RunInfo, p Progress) Decision {
|
|
mu.Lock()
|
|
calls++
|
|
mu.Unlock()
|
|
return Decision{ExtendBy: 50 * time.Millisecond, Nudge: []llm.Message{{Role: llm.RoleUser}}}
|
|
})
|
|
s := New(esc, 3)
|
|
s.checkInterval = 5 * time.Millisecond
|
|
h := s.Monitor(context.Background(), run.RunInfo{RunID: "r"}, 20*time.Millisecond)
|
|
defer h.Stop()
|
|
|
|
d0 := h.Deadline()
|
|
time.Sleep(60 * time.Millisecond) // cross the soft timeout with no activity
|
|
mu.Lock()
|
|
c := calls
|
|
mu.Unlock()
|
|
if c < 1 {
|
|
t.Fatalf("expected at least one escalation, got %d", c)
|
|
}
|
|
// Nudge was queued and is drained once.
|
|
if msgs := h.Steer(); len(msgs) == 0 {
|
|
t.Error("expected a queued steer nudge")
|
|
}
|
|
if msgs := h.Steer(); len(msgs) != 0 {
|
|
t.Error("steer should drain (be empty on second read)")
|
|
}
|
|
// Deadline was extended.
|
|
if !h.Deadline().After(d0) {
|
|
t.Error("deadline should have been extended past the original")
|
|
}
|
|
// A fresh step re-arms; another idle period escalates again.
|
|
h.RecordStep(1)
|
|
time.Sleep(60 * time.Millisecond)
|
|
mu.Lock()
|
|
c2 := calls
|
|
mu.Unlock()
|
|
if c2 <= c {
|
|
t.Errorf("a re-armed idle period should escalate again (%d -> %d)", c, c2)
|
|
}
|
|
}
|
|
|
|
func TestKillCollapsesDeadline(t *testing.T) {
|
|
esc := escFunc(func(context.Context, run.RunInfo, Progress) Decision {
|
|
return Decision{Kill: true, KillReason: "hung"}
|
|
})
|
|
s := New(esc, 10) // big backstop so only Kill collapses it
|
|
s.checkInterval = 5 * time.Millisecond
|
|
h := s.Monitor(context.Background(), run.RunInfo{RunID: "r"}, 20*time.Millisecond)
|
|
defer h.Stop()
|
|
time.Sleep(60 * time.Millisecond)
|
|
if h.Deadline().After(time.Now().Add(time.Second)) {
|
|
t.Error("Kill should collapse the deadline to ~now")
|
|
}
|
|
}
|
|
|
|
func TestExtendOnceOnlyFiresOnce(t *testing.T) {
|
|
e := &ExtendOnce{By: time.Minute}
|
|
// Same run id: only the first call extends.
|
|
d1 := e.OnSoftTimeout(context.Background(), run.RunInfo{RunID: "r1"}, Progress{})
|
|
d2 := e.OnSoftTimeout(context.Background(), run.RunInfo{RunID: "r1"}, Progress{})
|
|
if d1.ExtendBy != time.Minute {
|
|
t.Errorf("first decision should extend, got %+v", d1)
|
|
}
|
|
if d2.ExtendBy != 0 || d2.Kill {
|
|
t.Errorf("second call for the same run should be a no-op, got %+v", d2)
|
|
}
|
|
// A DIFFERENT run still gets its own one extension (per-run, not global).
|
|
if d3 := e.OnSoftTimeout(context.Background(), run.RunInfo{RunID: "r2"}, Progress{}); d3.ExtendBy != time.Minute {
|
|
t.Errorf("a different run should get its own extension, got %+v", d3)
|
|
}
|
|
}
|
|
|
|
func TestZeroSoftTimeoutNotMonitored(t *testing.T) {
|
|
s := New(nil, 3)
|
|
if h := s.Monitor(context.Background(), run.RunInfo{}, 0); h != nil {
|
|
t.Error("zero soft timeout should return a nil handle (not monitored)")
|
|
}
|
|
}
|