fix(agent): recover the front-loaded answer over a summary closer
A third degenerate terminal shape from the glm-5.2 cite pattern: the model
front-loads its full answer into the cite-call turn, then closes with a
bookkeeping ack plus a one-line compression ("Citations are logged. Short
version: ..."). mort run b3cb9ee9 delivered 151 chars of a 2,089-char
answer this way — the closer was neither a back-reference (over the 120
cap, no back-ref phrase) nor a citations addendum (no label-colon, no
links), so finalOutput let it stand.
isSummaryCloser keys on the ack sentence alone (the verb must end the
sentence, so prose about citations never matches; a compression marker
without the ack is deliberately out of scope), and the new modeSummary
recovery bar makes the 3x dwarf ratio mandatory at every length: unlike a
back-reference this closer carries real answer content, so it is only
displaced by the clearly-fuller original it compressed.
The citations/back-ref bool becomes a three-way recoveryMode; existing
behavior for both old modes is unchanged.
Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
+123
-1
@@ -72,6 +72,43 @@ func TestIsCitationsOnly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// b3cb9ee9Closer is the verbatim terminal turn from mort run b3cb9ee9: a
|
||||
// 2,089-char answer was front-loaded into the cite-call turn and this 151-char
|
||||
// compression was all that got delivered.
|
||||
const b3cb9ee9Closer = "Citations are logged. Short version: the bulk of that ~$64M was AIPAC and dark-money super PACs, not the party committees — and it still wasn't enough."
|
||||
|
||||
func TestIsSummaryCloser(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{"b3cb9ee9-verbatim", b3cb9ee9Closer, true},
|
||||
{"ack-only", "Citations are logged.", true},
|
||||
{"ack-no-copula", "Citations logged.", true},
|
||||
{"claims-cited", "All claims cited.", true},
|
||||
{"verb-first", "Logged the citations.", true},
|
||||
{"done-prefix", "Done — citations logged.", true},
|
||||
{"ack-then-tldr", "Sources have been recorded! TL;DR: the GPU was the bottleneck.", true},
|
||||
{"references-noted", "References noted. In short: yes, it ships Tuesday.", true},
|
||||
|
||||
{"empty", "", false},
|
||||
{"ack-continues-midsentence", "The citations are recorded in the court transcript, which shows the filing dates.", false},
|
||||
{"ack-verb-then-clause", "Citations are logged in Zotero whenever you click the save button.", false},
|
||||
{"compression-without-ack", "Short version: yes.", false}, // deliberately out of scope
|
||||
{"mentions-citations-midsentence", "The paper's citations are what got it retracted.", false},
|
||||
{"crisp-number", "42", false},
|
||||
{"over-cap", "Citations are logged. " + strings.Repeat("The long version has many more details worth keeping. ", 6), false}, // >300: too substantial to replace
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := isSummaryCloser(c.in); got != c.want {
|
||||
t.Errorf("isSummaryCloser(%q) = %v, want %v", c.in, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func asst(text string, tools ...llm.ToolCall) llm.Message {
|
||||
m := llm.Message{Role: llm.RoleAssistant}
|
||||
if text != "" {
|
||||
@@ -83,7 +120,8 @@ func asst(text string, tools ...llm.ToolCall) llm.Message {
|
||||
|
||||
func TestFinalOutput(t *testing.T) {
|
||||
cite := []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}}
|
||||
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200
|
||||
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200
|
||||
hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12)) // >3x the b3cb9ee9 closer
|
||||
// A sources/citations-only terminal — the glm-5.2 "cite" shape behind mort
|
||||
// issue #1418: the prose answer was front-loaded into the tool-call turn and
|
||||
// the terminal turn carried only the citations.
|
||||
@@ -256,6 +294,59 @@ func TestFinalOutput(t *testing.T) {
|
||||
terminal: sources,
|
||||
want: longConversationalAnswer + "\n\n" + sources,
|
||||
},
|
||||
{
|
||||
// The b3cb9ee9 shape: full answer front-loaded into the cite turn,
|
||||
// then a summary closer. The closer is discarded — its content is a
|
||||
// strict compression of the recovered answer.
|
||||
name: "summary closer discarded when the front-loaded answer dwarfs it",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("where did the $64M come from?"),
|
||||
asst(hugeAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(b3cb9ee9Closer),
|
||||
},
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: hugeAnswer,
|
||||
},
|
||||
{
|
||||
// The dwarf ratio is mandatory for a summary closer at EVERY length:
|
||||
// a prior turn that is longer but not clearly the fuller original
|
||||
// (here ~275 chars vs a 151-char closer, under the 3x bar) must not
|
||||
// displace a closer that carries real answer content.
|
||||
name: "summary closer kept when the prior turn does not dwarf it",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(b3cb9ee9Closer),
|
||||
},
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: b3cb9ee9Closer,
|
||||
},
|
||||
{
|
||||
// An ack-only closer ("Citations are logged.") is tiny, so even a
|
||||
// modest front-loaded answer clears the ratio and replaces it.
|
||||
name: "ack-only summary closer recovered over a modest answer",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst("Citations are logged."),
|
||||
},
|
||||
terminal: "Citations are logged.",
|
||||
want: longAnswer,
|
||||
},
|
||||
{
|
||||
name: "summary closer with only a preamble prior keeps the closer",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst("Let me gather the numbers.", cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(b3cb9ee9Closer),
|
||||
},
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: b3cb9ee9Closer,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
@@ -326,6 +417,37 @@ func TestRun_HealthyTerminalUnchanged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_RecoversFrontLoadedAnswerOverSummaryCloser reproduces mort run
|
||||
// b3cb9ee9 end-to-end: the model front-loads its full answer into the
|
||||
// cite-call turn, the cite results come back, and the terminal turn is only a
|
||||
// bookkeeping ack plus a one-line compression. The delivered output must be
|
||||
// the front-loaded answer, with no extra model call.
|
||||
func TestRun_RecoversFrontLoadedAnswerOverSummaryCloser(t *testing.T) {
|
||||
hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12))
|
||||
fp := fake.New("fp")
|
||||
fp.Enqueue("test-model",
|
||||
fake.ReplyWith(llm.Response{
|
||||
Parts: []llm.Part{llm.Text(hugeAnswer)},
|
||||
ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
|
||||
FinishReason: llm.FinishToolCalls,
|
||||
Usage: llm.Usage{InputTokens: 10, OutputTokens: 5},
|
||||
}),
|
||||
fake.Reply(b3cb9ee9Closer),
|
||||
)
|
||||
|
||||
a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
|
||||
res, err := a.Run(context.Background(), "where did the $64M come from?")
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if res.Output != hugeAnswer {
|
||||
t.Errorf("Output = %q, want recovered front-loaded answer", res.Output)
|
||||
}
|
||||
if n := len(fp.Calls()); n != 2 {
|
||||
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_RecoversFrontLoadedAnswerWithCitations reproduces mort issue #1418
|
||||
// end-to-end: the model front-loads the prose answer into the tool-call turn
|
||||
// and closes with a sources-only terminal turn. The delivered output must be
|
||||
|
||||
Reference in New Issue
Block a user