fix(run): the FinalGuard nudge round can never replace a non-blank answer
executus CI / test (pull_request) Successful in 49s

Live regression (mort run d5ec39f4, 2026-07-15): a research run's page-cache
file false-positived the guard, and the nudge round's meta closer ("my
answer above stands as-is") REPLACED the real answer — hosts deliver
Output text plus separately-drained attachments, so the user received the
closer and never saw the answer.

New merge policy: the nudge round exists to make tool calls (queue the
delivery); its final text is discarded and recorded as a final_nudge_result
audit event, so a round that declined to queue stays observable without
reaching the user. The only exception is a blank original stop turn, where
the nudge round's text is the only answer there is. Messages/Usage still
merge (audit, PostRun, checkpoint continuity).

Additive on top of 007b753 (mort main pins that commit as a pseudo-version;
this branch must never be rewritten).

Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
2026-07-15 10:15:44 -04:00
co-authored by Claude Fable 5
parent 007b7539c2
commit 7a074d8fa2
3 changed files with 89 additions and 18 deletions
+54 -13
View File
@@ -41,17 +41,21 @@ func (r *eventRecorder) LogEvent(eventType string, _ map[string]any) {
// TestFinalGuardNudgeRunsOneExtraRound is the core announce-then-stop fix: the
// model stops with "Done — sent!" (nothing delivered), the guard returns a
// nudge, and the SAME conversation runs one more round — here a tool call then
// a corrected final answer. The guard is consulted exactly once (the nudge
// round's own stop is final), and the audit log records the nudge.
// nudge, and the SAME conversation runs one more round — here a tool call
// (queueing the delivery) then a closing text turn. The guard is consulted
// exactly once (the nudge round's own stop is final), the audit log records
// the nudge, and the ORIGINAL answer text survives: the nudge round's closer
// is discarded (recorded as final_nudge_result), because hosts deliver Output
// text + separately-drained attachments — replacing Output with the closer is
// the run-d5ec39f4 lost-answer regression.
func TestFinalGuardNudgeRunsOneExtraRound(t *testing.T) {
fp := fake.New("fake")
fp.Enqueue("m",
fake.Reply("Done — sent you the file!"),
// Nudge round: an (unregistered → tolerated error result) delivery call,
// then the corrected final answer.
// then a meta closer that must NOT become the run output.
fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c1", Name: "send_attachments", Arguments: []byte(`{}`)}}}),
fake.Reply("delivered for real"),
fake.Reply("queued the file — answer above stands"),
)
m, _ := fp.Model("m")
rec := &eventRecorder{}
@@ -72,8 +76,11 @@ func TestFinalGuardNudgeRunsOneExtraRound(t *testing.T) {
if res.Err != nil {
t.Fatalf("run error: %v", res.Err)
}
if res.Output != "delivered for real" {
t.Fatalf("output = %q, want the nudge round's answer", res.Output)
if res.Output != "Done — sent you the file!" {
t.Fatalf("output = %q, want the ORIGINAL answer preserved (nudge closer discarded)", res.Output)
}
if n := fp.CallCount("m"); n != 3 {
t.Fatalf("model called %d times, want 3 (the nudge round must still run)", n)
}
if guard.calls != 1 {
t.Fatalf("guard consulted %d times, want exactly 1 (no re-guard of the nudge round)", guard.calls)
@@ -84,17 +91,51 @@ func TestFinalGuardNudgeRunsOneExtraRound(t *testing.T) {
if guard.state.Output != "Done — sent you the file!" {
t.Errorf("guard state.Output = %q, want the pre-nudge final text", guard.state.Output)
}
var sawNudgeEvent bool
var sawNudge, sawResult bool
for _, ev := range rec.events {
if ev == "final_nudge" {
sawNudgeEvent = true
switch ev {
case "final_nudge":
sawNudge = true
case "final_nudge_result":
sawResult = true
}
}
if !sawNudgeEvent {
if !sawNudge {
t.Errorf("audit events %v missing final_nudge", rec.events)
}
if rec.stats.Output != "delivered for real" {
t.Errorf("audit close output = %q, want the nudged output", rec.stats.Output)
if !sawResult {
t.Errorf("audit events %v missing final_nudge_result (the discarded closer must stay observable)", rec.events)
}
if rec.stats.Output != "Done — sent you the file!" {
t.Errorf("audit close output = %q, want the original answer", rec.stats.Output)
}
}
// TestFinalGuardNudgeFillsEmptyOriginal: when the ORIGINAL stop turn was blank
// (a degenerate run), the nudge round's text is the only answer there is — it
// becomes the output instead of being discarded.
func TestFinalGuardNudgeFillsEmptyOriginal(t *testing.T) {
fp := fake.New("fake")
fp.Enqueue("m",
fake.Reply(" \n"),
fake.Reply("actual answer from the nudge round"),
)
m, _ := fp.Model("m")
guard := &guardFunc{nudge: func(RunInfo, FinalState) string { return "deliver or correct" }}
ex := New(Config{
Registry: tool.NewRegistry(),
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
Ports: Ports{FinalGuard: guard},
})
res := ex.Run(context.Background(),
RunnableAgent{ModelTier: "m"}, tool.Invocation{RunID: "r"}, "hi")
if res.Err != nil {
t.Fatalf("run error: %v", res.Err)
}
if res.Output != "actual answer from the nudge round" {
t.Fatalf("output = %q, want the nudge round's text to fill a blank original", res.Output)
}
}