diff --git a/agent/finalize.go b/agent/finalize.go index 531d94f..c311b4a 100644 --- a/agent/finalize.go +++ b/agent/finalize.go @@ -14,8 +14,8 @@ import ( // their answer to the final, tool-free turn. But some models — notably several // open-weight ones — "front-load" their full answer into an earlier turn that // ALSO calls a tool (e.g. answer text alongside a citation call), then close -// with a degenerate terminal turn that is not itself the answer. Two shapes are -// recovered from the transcript (zero extra model calls): +// with a degenerate terminal turn that is not itself the answer. Three shapes +// are recovered from the transcript (zero extra model calls): // // - a trivial back-reference ("(Already answered above.)", "see above", …): // the real answer sits earlier, so recover it and DISCARD the worthless @@ -25,25 +25,39 @@ import ( // glm-5.2 "cite" pattern behind mort issue #1418). The citations are real, // useful content — unlike a back-reference — so recover the prior answer and // KEEP the citations, appended below it. +// - a bookkeeping closer ("Citations are logged. Short version: …"): the +// model acknowledged the citation round and compressed the answer it had +// already written into a one-liner (mort run b3cb9ee9 — a 2,089-char answer +// shrank to 151 chars at delivery). The compression is strictly poorer than +// the front-loaded answer, so recover the prior turn and DISCARD the +// closer — but only when the prior turn clearly dwarfs it, because unlike a +// back-reference this closer DOES carry answer content (see modeSummary). // -// A citations addendum is tested first and wins over the back-reference test (a -// short terminal can be both), so its links are never discarded. When the -// terminal text stands on its own it is returned unchanged; when it is -// degenerate but nothing better can be recovered, it is returned as-is (a bare -// sources list still beats nothing). +// A citations addendum is tested first and wins over the other two (a short +// terminal can match more than one shape), so its links are never discarded; +// the summary closer is tested before the plain weak-final test so the +// stricter recovery bar applies when both match. When the terminal text stands +// on its own it is returned unchanged; when it is degenerate but nothing +// better can be recovered, it is returned as-is (a compressed answer still +// beats nothing). // // msgs must already include the terminal assistant message as its last element // (the loop appends it before calling this); terminal is that message's text. func finalOutput(msgs []llm.Message, terminal string) string { - citations := isCitationsOnly(terminal) - if !citations && !isWeakFinal(terminal) { + mode := modeBackRef + switch { + case isCitationsOnly(terminal): + mode = modeCitations + case isSummaryCloser(terminal): + mode = modeSummary + case !isWeakFinal(terminal): return terminal } - rec, ok := lastSubstantiveAssistantText(msgs, terminal, citations) + rec, ok := lastSubstantiveAssistantText(msgs, terminal, mode) if !ok { return terminal } - if citations { + if mode == modeCitations { // Preserve the citations addendum below the recovered answer, unless the // recovered turn already carries it (guards against a duplicate sources // block when the front-loaded turn included its own citations). The @@ -65,11 +79,47 @@ func stripURLAngles(s string) string { return strings.NewReplacer("<", "", ">", "").Replace(s) } +// recoveryMode selects the bar a prior assistant turn must clear to replace +// the terminal turn (see isSubstantiveAnswer) and what finalOutput does with +// the terminal once recovery succeeds. +type recoveryMode int + +const ( + // modeBackRef: the terminal is empty or a pure back-reference — worthless + // on its own, so any real prior answer replaces it and it is discarded. + modeBackRef recoveryMode = iota + // modeCitations: the terminal is a sources-only addendum — not a rival + // answer, so the dwarf ratio is skipped and the addendum is kept, appended + // below the recovered answer. + modeCitations + // modeSummary: the terminal acknowledges the citation round and may carry + // a short compression of the front-loaded answer. Unlike a back-reference + // it DOES contain answer content, so it is only replaced when a prior turn + // clearly dwarfs it — the ratio is mandatory at every length, and the + // closer is discarded (its content is a strict subset of what it replaced). + modeSummary +) + // backRefRe matches a terminal turn that merely points back to an earlier // message instead of stating the answer ("(Already answered above.)", // "see above", "as I said", ...). var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(said|mentioned|stated|noted)|answered (that )?above|per my (previous|earlier))`) +// summaryCloserRe matches a terminal turn that OPENS with a bookkeeping +// acknowledgment of the citation round — "Citations are logged.", "Sources +// cited.", "Logged the citations." — the shape a model produces when it +// front-loaded its answer into an earlier cite-call turn and closes by +// acknowledging the tool results, often followed by a "Short version: …" +// compression of the answer it already wrote. The ack clause must end at a +// sentence terminator ([.!]) DIRECTLY after the verb: "The citations are +// recorded in the court transcript…" is a real answer about citations, not +// bookkeeping, and must never match. A compression marker without the ack +// ("Short version: no.") is deliberately out of scope — a user who asked for +// brevity would be answered with exactly that shape, and misclassifying it +// would hijack a legitimate answer; an unmatched closer merely keeps today's +// behavior (fail closed). +var summaryCloserRe = regexp.MustCompile(`(?i)^[\s>#*_-]*((done|all set|ok(ay)?)[\s,.!:—-]+)?(((all|the)\s+)?(citations?|sources?|references?|claims?)\s+((are|were|have\s+been|all)\s+)*(logged|recorded|cited|saved|noted|captured|filed)|logged\s+((all|the)\s+)*(citations?|sources?|references?))[.!]`) + // preambleRe matches intent-announcing prefixes ("Let me search...", "I'll // check...") so a preamble is never mistaken for the answer during recovery. var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]|sure[,. ]|okay[,. ]|on it|checking)`) @@ -114,6 +164,11 @@ const ( // be at most len/N of the whole, so a prose answer that merely opens with // "Source:" and cites a URL mid-sentence is not mistaken for a bare list. citationDominatedDivisor = 3 + // summaryCloserMaxChars bounds a summary closer: room for the ack sentence + // plus a couple of compression sentences (the b3cb9ee9 closer was 151). + // Beyond this the "short version" is substantial enough that replacing it + // risks losing content the front-loaded turn never had. + summaryCloserMaxChars = 300 ) // isWeakFinal reports whether a terminal turn's text fails to stand on its own @@ -152,11 +207,26 @@ func isCitationsOnly(s string) bool { return len(residue) <= len(t)/citationDominatedDivisor } +// isSummaryCloser reports whether a terminal turn is a bookkeeping closer: it +// opens with a complete "citations are logged"-style ack sentence (see +// summaryCloserRe) and is short enough that whatever follows the ack can only +// be a compression of an earlier, fuller answer. Whether that fuller answer +// actually exists is modeSummary's job — the dwarf ratio in +// isSubstantiveAnswer keeps a matching closer in place when nothing earlier +// clearly outweighs it. +func isSummaryCloser(s string) bool { + t := strings.TrimSpace(s) + if t == "" || len(t) > summaryCloserMaxChars { + return false + } + return summaryCloserRe.MatchString(t) +} + // lastSubstantiveAssistantText scans msgs newest→oldest (skipping the terminal // turn and empty tool-only turns) for the most recent assistant turn whose text -// reads like a real answer. citations selects the recovery bar (see +// reads like a real answer. mode selects the recovery bar (see // isSubstantiveAnswer). Returns ("", false) when nothing qualifies. -func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations bool) (string, bool) { +func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, mode recoveryMode) (string, bool) { tt := strings.TrimSpace(terminal) for i := len(msgs) - 1; i >= 0; i-- { m := msgs[i] @@ -167,7 +237,7 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations if txt == "" || txt == tt { continue // the terminal turn itself, or an empty tool-only turn } - if isSubstantiveAnswer(txt, tt, citations) { + if isSubstantiveAnswer(txt, tt, mode) { return txt, true } } @@ -177,20 +247,28 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations // isSubstantiveAnswer reports whether txt (a prior assistant turn) reads like a // real answer rather than a preamble, relative to the terminal text. // -// A sufficiently long turn (>= recoverMinChars) is accepted unconditionally: a -// multi-hundred-char turn is an answer even when it opens conversationally -// ("Sure, here's…", "Let me explain: …"), so the preamble filter is NOT applied -// to it — applying it there would drop a legitimate long front-loaded answer. -// Only in the borderline band does a turn have to clear a floor, not read like a -// short planning preamble ("Let me look that up…"), and — unless the terminal is -// a citations addendum (not a rival answer, so its length is irrelevant) — also -// clearly dwarf the terminal. -func isSubstantiveAnswer(txt, terminal string, citations bool) bool { +// modeSummary demands the dwarf ratio FIRST, at every length: a summary closer +// carries a real (compressed) answer, so replacing it is only justified when +// the prior turn is clearly the fuller original it was compressed from. +// +// A sufficiently long turn (>= recoverMinChars) is otherwise accepted +// unconditionally: a multi-hundred-char turn is an answer even when it opens +// conversationally ("Sure, here's…", "Let me explain: …"), so the preamble +// filter is NOT applied to it — applying it there would drop a legitimate long +// front-loaded answer. Only in the borderline band does a turn have to clear a +// floor, not read like a short planning preamble ("Let me look that up…"), +// and — for modeBackRef only — also clearly dwarf the terminal (a citations +// addendum is not a rival answer, so its length is irrelevant; a summary +// closer already proved the ratio above). +func isSubstantiveAnswer(txt, terminal string, mode recoveryMode) bool { + if mode == modeSummary && len(txt) < recoverRatio*len(terminal) { + return false + } if len(txt) >= recoverMinChars { return true } if len(txt) < recoverFloorChars || preambleRe.MatchString(txt) { return false } - return citations || len(txt) >= recoverRatio*len(terminal) + return mode != modeBackRef || len(txt) >= recoverRatio*len(terminal) } diff --git a/agent/finalize_test.go b/agent/finalize_test.go index a8948cb..cef84ef 100644 --- a/agent/finalize_test.go +++ b/agent/finalize_test.go @@ -72,6 +72,43 @@ func TestIsCitationsOnly(t *testing.T) { } } +// b3cb9ee9Closer is the verbatim terminal turn from mort run b3cb9ee9: a +// 2,089-char answer was front-loaded into the cite-call turn and this 151-char +// compression was all that got delivered. +const b3cb9ee9Closer = "Citations are logged. Short version: the bulk of that ~$64M was AIPAC and dark-money super PACs, not the party committees — and it still wasn't enough." + +func TestIsSummaryCloser(t *testing.T) { + cases := []struct { + name string + in string + want bool + }{ + {"b3cb9ee9-verbatim", b3cb9ee9Closer, true}, + {"ack-only", "Citations are logged.", true}, + {"ack-no-copula", "Citations logged.", true}, + {"claims-cited", "All claims cited.", true}, + {"verb-first", "Logged the citations.", true}, + {"done-prefix", "Done — citations logged.", true}, + {"ack-then-tldr", "Sources have been recorded! TL;DR: the GPU was the bottleneck.", true}, + {"references-noted", "References noted. In short: yes, it ships Tuesday.", true}, + + {"empty", "", false}, + {"ack-continues-midsentence", "The citations are recorded in the court transcript, which shows the filing dates.", false}, + {"ack-verb-then-clause", "Citations are logged in Zotero whenever you click the save button.", false}, + {"compression-without-ack", "Short version: yes.", false}, // deliberately out of scope + {"mentions-citations-midsentence", "The paper's citations are what got it retracted.", false}, + {"crisp-number", "42", false}, + {"over-cap", "Citations are logged. " + strings.Repeat("The long version has many more details worth keeping. ", 6), false}, // >300: too substantial to replace + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := isSummaryCloser(c.in); got != c.want { + t.Errorf("isSummaryCloser(%q) = %v, want %v", c.in, got, c.want) + } + }) + } +} + func asst(text string, tools ...llm.ToolCall) llm.Message { m := llm.Message{Role: llm.RoleAssistant} if text != "" { @@ -83,7 +120,8 @@ func asst(text string, tools ...llm.ToolCall) llm.Message { func TestFinalOutput(t *testing.T) { cite := []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}} - longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200 + longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200 + hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12)) // >3x the b3cb9ee9 closer // A sources/citations-only terminal — the glm-5.2 "cite" shape behind mort // issue #1418: the prose answer was front-loaded into the tool-call turn and // the terminal turn carried only the citations. @@ -256,6 +294,59 @@ func TestFinalOutput(t *testing.T) { terminal: sources, want: longConversationalAnswer + "\n\n" + sources, }, + { + // The b3cb9ee9 shape: full answer front-loaded into the cite turn, + // then a summary closer. The closer is discarded — its content is a + // strict compression of the recovered answer. + name: "summary closer discarded when the front-loaded answer dwarfs it", + msgs: []llm.Message{ + llm.UserText("where did the $64M come from?"), + asst(hugeAnswer, cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst(b3cb9ee9Closer), + }, + terminal: b3cb9ee9Closer, + want: hugeAnswer, + }, + { + // The dwarf ratio is mandatory for a summary closer at EVERY length: + // a prior turn that is longer but not clearly the fuller original + // (here ~275 chars vs a 151-char closer, under the 3x bar) must not + // displace a closer that carries real answer content. + name: "summary closer kept when the prior turn does not dwarf it", + msgs: []llm.Message{ + llm.UserText("q?"), + asst(longAnswer, cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst(b3cb9ee9Closer), + }, + terminal: b3cb9ee9Closer, + want: b3cb9ee9Closer, + }, + { + // An ack-only closer ("Citations are logged.") is tiny, so even a + // modest front-loaded answer clears the ratio and replaces it. + name: "ack-only summary closer recovered over a modest answer", + msgs: []llm.Message{ + llm.UserText("q?"), + asst(longAnswer, cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst("Citations are logged."), + }, + terminal: "Citations are logged.", + want: longAnswer, + }, + { + name: "summary closer with only a preamble prior keeps the closer", + msgs: []llm.Message{ + llm.UserText("q?"), + asst("Let me gather the numbers.", cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst(b3cb9ee9Closer), + }, + terminal: b3cb9ee9Closer, + want: b3cb9ee9Closer, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -326,6 +417,37 @@ func TestRun_HealthyTerminalUnchanged(t *testing.T) { } } +// TestRun_RecoversFrontLoadedAnswerOverSummaryCloser reproduces mort run +// b3cb9ee9 end-to-end: the model front-loads its full answer into the +// cite-call turn, the cite results come back, and the terminal turn is only a +// bookkeeping ack plus a one-line compression. The delivered output must be +// the front-loaded answer, with no extra model call. +func TestRun_RecoversFrontLoadedAnswerOverSummaryCloser(t *testing.T) { + hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12)) + fp := fake.New("fp") + fp.Enqueue("test-model", + fake.ReplyWith(llm.Response{ + Parts: []llm.Part{llm.Text(hugeAnswer)}, + ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}}, + FinishReason: llm.FinishToolCalls, + Usage: llm.Usage{InputTokens: 10, OutputTokens: 5}, + }), + fake.Reply(b3cb9ee9Closer), + ) + + a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t))) + res, err := a.Run(context.Background(), "where did the $64M come from?") + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.Output != hugeAnswer { + t.Errorf("Output = %q, want recovered front-loaded answer", res.Output) + } + if n := len(fp.Calls()); n != 2 { + t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n) + } +} + // TestRun_RecoversFrontLoadedAnswerWithCitations reproduces mort issue #1418 // end-to-end: the model front-loads the prose answer into the tool-call turn // and closes with a sources-only terminal turn. The delivered output must be