fix(run): the FinalGuard nudge round can never replace a non-blank answer
executus CI / test (pull_request) Successful in 49s
executus CI / test (pull_request) Successful in 49s
Live regression (mort run d5ec39f4, 2026-07-15): a research run's page-cache
file false-positived the guard, and the nudge round's meta closer ("my
answer above stands as-is") REPLACED the real answer — hosts deliver
Output text plus separately-drained attachments, so the user received the
closer and never saw the answer.
New merge policy: the nudge round exists to make tool calls (queue the
delivery); its final text is discarded and recorded as a final_nudge_result
audit event, so a round that declined to queue stays observable without
reaching the user. The only exception is a blank original stop turn, where
the nudge round's text is the only answer there is. Messages/Usage still
merge (audit, PostRun, checkpoint continuity).
Additive on top of 007b753 (mort main pins that commit as a pseudo-version;
this branch must never be rewritten).
Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
+54
-13
@@ -41,17 +41,21 @@ func (r *eventRecorder) LogEvent(eventType string, _ map[string]any) {
|
||||
|
||||
// TestFinalGuardNudgeRunsOneExtraRound is the core announce-then-stop fix: the
|
||||
// model stops with "Done — sent!" (nothing delivered), the guard returns a
|
||||
// nudge, and the SAME conversation runs one more round — here a tool call then
|
||||
// a corrected final answer. The guard is consulted exactly once (the nudge
|
||||
// round's own stop is final), and the audit log records the nudge.
|
||||
// nudge, and the SAME conversation runs one more round — here a tool call
|
||||
// (queueing the delivery) then a closing text turn. The guard is consulted
|
||||
// exactly once (the nudge round's own stop is final), the audit log records
|
||||
// the nudge, and the ORIGINAL answer text survives: the nudge round's closer
|
||||
// is discarded (recorded as final_nudge_result), because hosts deliver Output
|
||||
// text + separately-drained attachments — replacing Output with the closer is
|
||||
// the run-d5ec39f4 lost-answer regression.
|
||||
func TestFinalGuardNudgeRunsOneExtraRound(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.Reply("Done — sent you the file!"),
|
||||
// Nudge round: an (unregistered → tolerated error result) delivery call,
|
||||
// then the corrected final answer.
|
||||
// then a meta closer that must NOT become the run output.
|
||||
fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c1", Name: "send_attachments", Arguments: []byte(`{}`)}}}),
|
||||
fake.Reply("delivered for real"),
|
||||
fake.Reply("queued the file — answer above stands"),
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
rec := &eventRecorder{}
|
||||
@@ -72,8 +76,11 @@ func TestFinalGuardNudgeRunsOneExtraRound(t *testing.T) {
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "delivered for real" {
|
||||
t.Fatalf("output = %q, want the nudge round's answer", res.Output)
|
||||
if res.Output != "Done — sent you the file!" {
|
||||
t.Fatalf("output = %q, want the ORIGINAL answer preserved (nudge closer discarded)", res.Output)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 3 {
|
||||
t.Fatalf("model called %d times, want 3 (the nudge round must still run)", n)
|
||||
}
|
||||
if guard.calls != 1 {
|
||||
t.Fatalf("guard consulted %d times, want exactly 1 (no re-guard of the nudge round)", guard.calls)
|
||||
@@ -84,17 +91,51 @@ func TestFinalGuardNudgeRunsOneExtraRound(t *testing.T) {
|
||||
if guard.state.Output != "Done — sent you the file!" {
|
||||
t.Errorf("guard state.Output = %q, want the pre-nudge final text", guard.state.Output)
|
||||
}
|
||||
var sawNudgeEvent bool
|
||||
var sawNudge, sawResult bool
|
||||
for _, ev := range rec.events {
|
||||
if ev == "final_nudge" {
|
||||
sawNudgeEvent = true
|
||||
switch ev {
|
||||
case "final_nudge":
|
||||
sawNudge = true
|
||||
case "final_nudge_result":
|
||||
sawResult = true
|
||||
}
|
||||
}
|
||||
if !sawNudgeEvent {
|
||||
if !sawNudge {
|
||||
t.Errorf("audit events %v missing final_nudge", rec.events)
|
||||
}
|
||||
if rec.stats.Output != "delivered for real" {
|
||||
t.Errorf("audit close output = %q, want the nudged output", rec.stats.Output)
|
||||
if !sawResult {
|
||||
t.Errorf("audit events %v missing final_nudge_result (the discarded closer must stay observable)", rec.events)
|
||||
}
|
||||
if rec.stats.Output != "Done — sent you the file!" {
|
||||
t.Errorf("audit close output = %q, want the original answer", rec.stats.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFinalGuardNudgeFillsEmptyOriginal: when the ORIGINAL stop turn was blank
|
||||
// (a degenerate run), the nudge round's text is the only answer there is — it
|
||||
// becomes the output instead of being discarded.
|
||||
func TestFinalGuardNudgeFillsEmptyOriginal(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.Reply(" \n"),
|
||||
fake.Reply("actual answer from the nudge round"),
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
guard := &guardFunc{nudge: func(RunInfo, FinalState) string { return "deliver or correct" }}
|
||||
ex := New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: Ports{FinalGuard: guard},
|
||||
})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ModelTier: "m"}, tool.Invocation{RunID: "r"}, "hi")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "actual answer from the nudge round" {
|
||||
t.Fatalf("output = %q, want the nudge round's text to fill a blank original", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user