Merge pull request 'fix(agent): recover the front-loaded answer over a summary closer' (#25) from fix/finalize-summary-closer into main

agent/finalize.go

@@ -14,8 +14,8 @@ import (
 // their answer to the final, tool-free turn. But some models — notably several
 // open-weight ones — "front-load" their full answer into an earlier turn that
 // ALSO calls a tool (e.g. answer text alongside a citation call), then close
-// with a degenerate terminal turn that is not itself the answer. Two shapes are
-// recovered from the transcript (zero extra model calls):
+// with a degenerate terminal turn that is not itself the answer. Three shapes
+// are recovered from the transcript (zero extra model calls):
 //
 //   - a trivial back-reference ("(Already answered above.)", "see above", …):
 //     the real answer sits earlier, so recover it and DISCARD the worthless
@@ -25,25 +25,44 @@ import (
 //     glm-5.2 "cite" pattern behind mort issue #1418). The citations are real,
 //     useful content — unlike a back-reference — so recover the prior answer and
 //     KEEP the citations, appended below it.
+//   - a bookkeeping closer ("Citations are logged. Short version: …"): the
+//     model acknowledged the citation round and compressed the answer it had
+//     already written into a one-liner (mort run b3cb9ee9 — a 2,089-char answer
+//     shrank to a 153-byte closer at delivery). The compression is strictly
+//     poorer than the front-loaded answer, so recover the prior turn and
+//     DISCARD the closer — but only when the prior turn clearly dwarfs it,
+//     because unlike a back-reference this closer DOES carry answer content
+//     (see modeSummary).
 //
-// A citations addendum is tested first and wins over the back-reference test (a
-// short terminal can be both), so its links are never discarded. When the
-// terminal text stands on its own it is returned unchanged; when it is
-// degenerate but nothing better can be recovered, it is returned as-is (a bare
-// sources list still beats nothing).
+// A citations addendum is tested first and wins over the other two (a short
+// terminal can match more than one shape), so its links are never discarded.
+// The back-reference test wins over the summary-closer test: a terminal
+// matching both ("Citations are logged. As I said above…") carries no answer
+// content of its own, so the looser back-ref recovery bar — not the summary
+// closer's dwarf ratio — is the right one. When the terminal text stands
+// on its own it is returned unchanged; when it is degenerate but nothing
+// better can be recovered, it is returned as-is (a compressed answer still
+// beats nothing).
 //
 // msgs must already include the terminal assistant message as its last element
 // (the loop appends it before calling this); terminal is that message's text.
 func finalOutput(msgs []llm.Message, terminal string) string {
-	citations := isCitationsOnly(terminal)
-	if !citations && !isWeakFinal(terminal) {
+	mode := modeBackRef
+	switch {
+	case isCitationsOnly(terminal):
+		mode = modeCitations
+	case isWeakFinal(terminal):
+		mode = modeBackRef
+	case isSummaryCloser(terminal):
+		mode = modeSummary
+	default:
 		return terminal
 	}
-	rec, ok := lastSubstantiveAssistantText(msgs, terminal, citations)
+	rec, ok := lastSubstantiveAssistantText(msgs, terminal, mode)
 	if !ok {
 		return terminal
 	}
-	if citations {
+	if mode == modeCitations {
 		// Preserve the citations addendum below the recovered answer, unless the
 		// recovered turn already carries it (guards against a duplicate sources
 		// block when the front-loaded turn included its own citations). The
@@ -65,11 +84,64 @@ func stripURLAngles(s string) string {
 	return strings.NewReplacer("<", "", ">", "").Replace(s)
 }
 
+// recoveryMode selects the bar a prior assistant turn must clear to replace
+// the terminal turn (see isSubstantiveAnswer) and what finalOutput does with
+// the terminal once recovery succeeds.
+type recoveryMode int
+
+const (
+	// modeBackRef: the terminal is empty or a pure back-reference — worthless
+	// on its own, so any real prior answer replaces it and it is discarded.
+	modeBackRef recoveryMode = iota
+	// modeCitations: the terminal is a sources-only addendum — not a rival
+	// answer, so the dwarf ratio is skipped and the addendum is kept, appended
+	// below the recovered answer.
+	modeCitations
+	// modeSummary: the terminal acknowledges the citation round and may carry
+	// a short compression of the front-loaded answer. Unlike a back-reference
+	// it DOES contain answer content, so it is only replaced when a prior turn
+	// clearly dwarfs it — the ratio is mandatory at every length, the recovery
+	// scan stops at the most recent user message (a compression can only be of
+	// THIS turn's answer; never resurrect one from an earlier question), and
+	// the closer is discarded (its content is a strict subset of what it
+	// replaced).
+	modeSummary
+)
+
 // backRefRe matches a terminal turn that merely points back to an earlier
 // message instead of stating the answer ("(Already answered above.)",
 // "see above", "as I said", ...).
 var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(said|mentioned|stated|noted)|answered (that )?above|per my (previous|earlier))`)
 
+// summaryCloserRe matches a terminal turn that OPENS with a bookkeeping
+// acknowledgment of the citation round — "Citations are logged.", "Sources
+// cited.", "Logged the citations." — the shape a model produces when it
+// front-loaded its answer into an earlier cite-call turn and closes by
+// acknowledging the tool results, often followed by a "Short version: …"
+// compression of the answer it already wrote. The ack clause must end at a
+// sentence terminator ([.!]) DIRECTLY after the verb: "The citations are
+// recorded in the court transcript…" is a real answer about citations, not
+// bookkeeping, and must never match. A compression marker without the ack
+// ("Short version: no.") is deliberately out of scope — a user who asked for
+// brevity would be answered with exactly that shape, and misclassifying it
+// would hijack a legitimate answer; an unmatched closer merely keeps today's
+// behavior (fail closed). Assembled from named fragments so the alternations
+// stay legible and extendable.
+const (
+	summaryPreface = `((done|all set|ok(ay)?)[\s,.!:—-]+)?` // optional "Done —" style opener
+	summaryNouns   = `(citations?|sources?|references?|claims?)`
+	// "all" appears here AND in summaryArticle on purpose: as a quantifier
+	// between noun and verb ("Citations all logged.") and as a determiner
+	// before the noun ("All claims cited.", "Logged all the citations.").
+	summaryCopulas = `((are|were|have\s+been|all)\s+)*`
+	summaryVerbs   = `(logged|recorded|cited|saved|noted|captured|filed)`
+	summaryArticle = `((all|the)\s+)*` // star, not ?: "Logged all the citations."
+)
+
+var summaryCloserRe = regexp.MustCompile(`(?i)^` + leadingMarkers + summaryPreface +
+	`(` + summaryArticle + summaryNouns + `\s+` + summaryCopulas + summaryVerbs +
+	`|logged\s+` + summaryArticle + summaryNouns + `)[.!]`)
+
 // preambleRe matches intent-announcing prefixes ("Let me search...", "I'll
 // check...") so a preamble is never mistaken for the answer during recovery.
 var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]|sure[,. ]|okay[,. ]|on it|checking)`)
@@ -83,7 +155,15 @@ var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]|
 // the colon/dash separator. Anchored at ^ so a normal answer that merely
 // mentions "sources" mid-sentence, or ends with a "Sources:" section AFTER its
 // prose, is never matched.
-var citationLabelRe = regexp.MustCompile(`(?i)^[\s>#*_+-]*(sources?|references?|citations?|works cited|further reading)\b[\s*_]*[:：\-—]`)
+var citationLabelRe = regexp.MustCompile(`(?i)^` + leadingMarkers +
+	`(sources?|references?|citations?|works cited|further reading)\b[\s*_]*[:：\-—]`)
+
+// leadingMarkers tolerates markdown noise before a label: emphasis (*, _),
+// list (-, +, *), block-quote (>), and ATX-heading (#) markers, with their
+// whitespace. Shared by citationLabelRe and summaryCloserRe so the two
+// classifiers cannot drift apart (the first draft of the summary class
+// dropped '+' by hand-copying this set).
+const leadingMarkers = `[\s>#*_+-]*`
 
 // linkRe matches a whole markdown link "[label](url)" or a bare URL. Used both
 // to require that a citations terminal carries at least one link and to strip
@@ -114,6 +194,12 @@ const (
 	// be at most len/N of the whole, so a prose answer that merely opens with
 	// "Source:" and cites a URL mid-sentence is not mistaken for a bare list.
 	citationDominatedDivisor = 3
+	// summaryCloserMaxChars bounds a summary closer: room for the ack sentence
+	// plus a couple of compression sentences (the b3cb9ee9 closer was 153
+	// bytes — Go len(), which is what every threshold here compares). Beyond
+	// this the "short version" is substantial enough that replacing it risks
+	// losing content the front-loaded turn never had.
+	summaryCloserMaxChars = 300
 )
 
 // isWeakFinal reports whether a terminal turn's text fails to stand on its own
@@ -152,14 +238,39 @@ func isCitationsOnly(s string) bool {
 	return len(residue) <= len(t)/citationDominatedDivisor
 }
 
+// isSummaryCloser reports whether a terminal turn is a bookkeeping closer: it
+// opens with a complete "citations are logged"-style ack sentence (see
+// summaryCloserRe) and is short enough that whatever follows the ack can only
+// be a compression of an earlier, fuller answer. Whether that fuller answer
+// actually exists is modeSummary's job — the dwarf ratio in
+// isSubstantiveAnswer keeps a matching closer in place when nothing earlier
+// clearly outweighs it.
+func isSummaryCloser(s string) bool {
+	t := strings.TrimSpace(s)
+	if t == "" || len(t) > summaryCloserMaxChars {
+		return false
+	}
+	return summaryCloserRe.MatchString(t)
+}
+
 // lastSubstantiveAssistantText scans msgs newest→oldest (skipping the terminal
 // turn and empty tool-only turns) for the most recent assistant turn whose text
-// reads like a real answer. citations selects the recovery bar (see
+// reads like a real answer. mode selects the recovery bar (see
 // isSubstantiveAnswer). Returns ("", false) when nothing qualifies.
-func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations bool) (string, bool) {
+func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, mode recoveryMode) (string, bool) {
 	tt := strings.TrimSpace(terminal)
 	for i := len(msgs) - 1; i >= 0; i-- {
 		m := msgs[i]
+		if mode == modeSummary && m.Role == llm.RoleUser {
+			// A summary closer compresses THIS turn's front-loaded answer, so
+			// the scan must not cross into an earlier question: once the dwarf
+			// ratio has rejected the current turn's text, walking further back
+			// would resurrect a stale answer to a DIFFERENT question — strictly
+			// worse than keeping the closer. (A mid-run steer message is also a
+			// user-role boundary; recovery then fails closed, which is fine.)
+			// The other modes keep their historical unbounded scan.
+			break
+		}
 		if m.Role != llm.RoleAssistant {
 			continue
 		}
@@ -167,7 +278,7 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations
 		if txt == "" || txt == tt {
 			continue // the terminal turn itself, or an empty tool-only turn
 		}
-		if isSubstantiveAnswer(txt, tt, citations) {
+		if isSubstantiveAnswer(txt, tt, mode) {
 			return txt, true
 		}
 	}
@@ -177,20 +288,29 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations
 // isSubstantiveAnswer reports whether txt (a prior assistant turn) reads like a
 // real answer rather than a preamble, relative to the terminal text.
 //
-// A sufficiently long turn (>= recoverMinChars) is accepted unconditionally: a
-// multi-hundred-char turn is an answer even when it opens conversationally
-// ("Sure, here's…", "Let me explain: …"), so the preamble filter is NOT applied
-// to it — applying it there would drop a legitimate long front-loaded answer.
-// Only in the borderline band does a turn have to clear a floor, not read like a
-// short planning preamble ("Let me look that up…"), and — unless the terminal is
-// a citations addendum (not a rival answer, so its length is irrelevant) — also
-// clearly dwarf the terminal.
-func isSubstantiveAnswer(txt, terminal string, citations bool) bool {
+// modeSummary demands the dwarf ratio FIRST, at every length: a summary closer
+// carries a real (compressed) answer, so replacing it is only justified when
+// the prior turn is clearly the fuller original it was compressed from.
+//
+// A sufficiently long turn (>= recoverMinChars) is otherwise accepted
+// unconditionally: a multi-hundred-char turn is an answer even when it opens
+// conversationally ("Sure, here's…", "Let me explain: …"), so the preamble
+// filter is NOT applied to it — applying it there would drop a legitimate long
+// front-loaded answer. Only in the borderline band does a turn have to clear a
+// floor, not read like a short planning preamble ("Let me look that up…"),
+// and — for modeBackRef only — also clearly dwarf the terminal (a citations
+// addendum is not a rival answer, so its length is irrelevant; a summary
+// closer already proved the ratio above).
+func isSubstantiveAnswer(txt, terminal string, mode recoveryMode) bool {
+	dwarfs := len(txt) >= recoverRatio*len(terminal)
+	if mode == modeSummary && !dwarfs {
+		return false
+	}
 	if len(txt) >= recoverMinChars {
 		return true
 	}
 	if len(txt) < recoverFloorChars || preambleRe.MatchString(txt) {
 		return false
 	}
-	return citations || len(txt) >= recoverRatio*len(terminal)
+	return mode != modeBackRef || dwarfs
 }

agent/finalize_test.go

@@ -72,6 +72,46 @@ func TestIsCitationsOnly(t *testing.T) {
 	}
 }
 
+// b3cb9ee9Closer is the verbatim terminal turn from mort run b3cb9ee9: a
+// 2,089-char answer was front-loaded into the cite-call turn and this 153-byte
+// compression (151 runes — the em dash is 3 bytes, and byte length is what the
+// thresholds compare) was all that got delivered.
+const b3cb9ee9Closer = "Citations are logged. Short version: the bulk of that ~$64M was AIPAC and dark-money super PACs, not the party committees — and it still wasn't enough."
+
+func TestIsSummaryCloser(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want bool
+	}{
+		{"b3cb9ee9-verbatim", b3cb9ee9Closer, true},
+		{"ack-only", "Citations are logged.", true},
+		{"ack-no-copula", "Citations logged.", true},
+		{"claims-cited", "All claims cited.", true},
+		{"verb-first", "Logged the citations.", true},
+		{"done-prefix", "Done — citations logged.", true},
+		{"ack-then-tldr", "Sources have been recorded! TL;DR: the GPU was the bottleneck.", true},
+		{"references-noted", "References noted. In short: yes, it ships Tuesday.", true},
+		{"plus-list-marker", "+ Citations are logged.", true},
+		{"logged-all-the", "Logged all the citations.", true},
+
+		{"empty", "", false},
+		{"ack-continues-midsentence", "The citations are recorded in the court transcript, which shows the filing dates.", false},
+		{"ack-verb-then-clause", "Citations are logged in Zotero whenever you click the save button.", false},
+		{"compression-without-ack", "Short version: yes.", false}, // deliberately out of scope
+		{"mentions-citations-midsentence", "The paper's citations are what got it retracted.", false},
+		{"crisp-number", "42", false},
+		{"over-cap", "Citations are logged. " + strings.Repeat("The long version has many more details worth keeping. ", 6), false}, // >300: too substantial to replace
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if got := isSummaryCloser(c.in); got != c.want {
+				t.Errorf("isSummaryCloser(%q) = %v, want %v", c.in, got, c.want)
+			}
+		})
+	}
+}
+
 func asst(text string, tools ...llm.ToolCall) llm.Message {
 	m := llm.Message{Role: llm.RoleAssistant}
 	if text != "" {
@@ -83,7 +123,8 @@ func asst(text string, tools ...llm.ToolCall) llm.Message {
 
 func TestFinalOutput(t *testing.T) {
 	cite := []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}}
-	longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200
+	longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6))  // >200
+	hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12)) // >3x the b3cb9ee9 closer
 	// A sources/citations-only terminal — the glm-5.2 "cite" shape behind mort
 	// issue #1418: the prose answer was front-loaded into the tool-call turn and
 	// the terminal turn carried only the citations.
@@ -103,6 +144,9 @@ func TestFinalOutput(t *testing.T) {
 	// A >=200-byte real answer that merely OPENS with a conversational word
 	// ("Sure,"). The preamble filter must NOT veto it (gadfly regression guard).
 	longConversationalAnswer := "Sure, here's the rundown: it currently sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle, with the sealed Founders Edition commanding the top of that range while used AIB cards go a bit lower."
+	// Matches BOTH the summary ack and backRefRe, within the 120-byte weak cap,
+	// and long enough (>~92 bytes) that longAnswer would fail the summary bar.
+	bothMatchCloser := "Citations are logged. As I mentioned above, the full detail on the money sources is in my earlier message."
 
 	tests := []struct {
 		name     string
@@ -256,6 +300,109 @@ func TestFinalOutput(t *testing.T) {
 			terminal: sources,
 			want:     longConversationalAnswer + "\n\n" + sources,
 		},
+		{
+			// The b3cb9ee9 shape: full answer front-loaded into the cite turn,
+			// then a summary closer. The closer is discarded — its content is a
+			// strict compression of the recovered answer.
+			name: "summary closer discarded when the front-loaded answer dwarfs it",
+			msgs: []llm.Message{
+				llm.UserText("where did the $64M come from?"),
+				asst(hugeAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(b3cb9ee9Closer),
+			},
+			terminal: b3cb9ee9Closer,
+			want:     hugeAnswer,
+		},
+		{
+			// The dwarf ratio is mandatory for a summary closer at EVERY length:
+			// a prior turn that is longer but not clearly the fuller original
+			// (here ~275 chars vs a 151-char closer, under the 3x bar) must not
+			// displace a closer that carries real answer content.
+			name: "summary closer kept when the prior turn does not dwarf it",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst(longAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(b3cb9ee9Closer),
+			},
+			terminal: b3cb9ee9Closer,
+			want:     b3cb9ee9Closer,
+		},
+		{
+			// An ack-only closer ("Citations are logged.") is tiny, so even a
+			// modest front-loaded answer clears the ratio and replaces it.
+			name: "ack-only summary closer recovered over a modest answer",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst(longAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst("Citations are logged."),
+			},
+			terminal: "Citations are logged.",
+			want:     longAnswer,
+		},
+		{
+			name: "summary closer with only a preamble prior keeps the closer",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst("Let me gather the numbers.", cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(b3cb9ee9Closer),
+			},
+			terminal: b3cb9ee9Closer,
+			want:     b3cb9ee9Closer,
+		},
+		{
+			// The modeSummary scan must stop at the most recent user message.
+			// Here the current turn's answer sits in the 1x-3x band (rejected
+			// by the ratio) while a dwarfing answer to a DIFFERENT question
+			// sits in history — resurrecting it would be strictly worse than
+			// keeping the closer.
+			name: "summary closer never resurrects a stale answer across the user boundary",
+			msgs: []llm.Message{
+				llm.UserText("earlier, unrelated question?"),
+				asst(hugeAnswer),
+				llm.UserText("q?"),
+				asst(conciseAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(b3cb9ee9Closer),
+			},
+			terminal: b3cb9ee9Closer,
+			want:     b3cb9ee9Closer,
+		},
+		{
+			// The boundary must not break the legitimate multi-turn case: the
+			// dwarfing front-loaded answer in THIS turn's window is recovered
+			// even with history behind it.
+			name: "summary closer recovery still works with history present",
+			msgs: []llm.Message{
+				llm.UserText("earlier, unrelated question?"),
+				asst(longAnswer),
+				llm.UserText("q?"),
+				asst(hugeAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(b3cb9ee9Closer),
+			},
+			terminal: b3cb9ee9Closer,
+			want:     hugeAnswer,
+		},
+		{
+			// A closer matching BOTH the ack shape and a back-reference
+			// carries no answer content, so the back-ref test must win and the
+			// ordinary recovery bar apply — under the summary bar this
+			// ~106-byte terminal would demand a ~318-byte prior and wrongly
+			// keep the closer over longAnswer.
+			name: "back-reference wins over the summary ack when both match",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst(longAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(bothMatchCloser),
+			},
+			terminal: bothMatchCloser,
+			want:     longAnswer,
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
@@ -326,6 +473,37 @@ func TestRun_HealthyTerminalUnchanged(t *testing.T) {
 	}
 }
 
+// TestRun_RecoversFrontLoadedAnswerOverSummaryCloser reproduces mort run
+// b3cb9ee9 end-to-end: the model front-loads its full answer into the
+// cite-call turn, the cite results come back, and the terminal turn is only a
+// bookkeeping ack plus a one-line compression. The delivered output must be
+// the front-loaded answer, with no extra model call.
+func TestRun_RecoversFrontLoadedAnswerOverSummaryCloser(t *testing.T) {
+	hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12))
+	fp := fake.New("fp")
+	fp.Enqueue("test-model",
+		fake.ReplyWith(llm.Response{
+			Parts:        []llm.Part{llm.Text(hugeAnswer)},
+			ToolCalls:    []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
+			FinishReason: llm.FinishToolCalls,
+			Usage:        llm.Usage{InputTokens: 10, OutputTokens: 5},
+		}),
+		fake.Reply(b3cb9ee9Closer),
+	)
+
+	a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
+	res, err := a.Run(context.Background(), "where did the $64M come from?")
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if res.Output != hugeAnswer {
+		t.Errorf("Output = %q, want recovered front-loaded answer", res.Output)
+	}
+	if n := len(fp.Calls()); n != 2 {
+		t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
+	}
+}
+
 // TestRun_RecoversFrontLoadedAnswerWithCitations reproduces mort issue #1418
 // end-to-end: the model front-loads the prose answer into the tool-call turn
 // and closes with a sources-only terminal turn. The delivered output must be