From 127966bb3ab3490e4aad6a8e2f6788fdc6bbf855 Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Wed, 5 Aug 2026 08:47:40 -0400 Subject: [PATCH 1/3] fix(agent): recover the front-loaded answer over a summary closer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A third degenerate terminal shape from the glm-5.2 cite pattern: the model front-loads its full answer into the cite-call turn, then closes with a bookkeeping ack plus a one-line compression ("Citations are logged. Short version: ..."). mort run b3cb9ee9 delivered 151 chars of a 2,089-char answer this way — the closer was neither a back-reference (over the 120 cap, no back-ref phrase) nor a citations addendum (no label-colon, no links), so finalOutput let it stand. isSummaryCloser keys on the ack sentence alone (the verb must end the sentence, so prose about citations never matches; a compression marker without the ack is deliberately out of scope), and the new modeSummary recovery bar makes the 3x dwarf ratio mandatory at every length: unlike a back-reference this closer carries real answer content, so it is only displaced by the clearly-fuller original it compressed. The citations/back-ref bool becomes a three-way recoveryMode; existing behavior for both old modes is unchanged. Co-Authored-By: Claude Fable 5 --- agent/finalize.go | 126 +++++++++++++++++++++++++++++++++-------- agent/finalize_test.go | 124 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 225 insertions(+), 25 deletions(-) diff --git a/agent/finalize.go b/agent/finalize.go index 531d94f..c311b4a 100644 --- a/agent/finalize.go +++ b/agent/finalize.go @@ -14,8 +14,8 @@ import ( // their answer to the final, tool-free turn. But some models — notably several // open-weight ones — "front-load" their full answer into an earlier turn that // ALSO calls a tool (e.g. answer text alongside a citation call), then close -// with a degenerate terminal turn that is not itself the answer. Two shapes are -// recovered from the transcript (zero extra model calls): +// with a degenerate terminal turn that is not itself the answer. Three shapes +// are recovered from the transcript (zero extra model calls): // // - a trivial back-reference ("(Already answered above.)", "see above", …): // the real answer sits earlier, so recover it and DISCARD the worthless @@ -25,25 +25,39 @@ import ( // glm-5.2 "cite" pattern behind mort issue #1418). The citations are real, // useful content — unlike a back-reference — so recover the prior answer and // KEEP the citations, appended below it. +// - a bookkeeping closer ("Citations are logged. Short version: …"): the +// model acknowledged the citation round and compressed the answer it had +// already written into a one-liner (mort run b3cb9ee9 — a 2,089-char answer +// shrank to 151 chars at delivery). The compression is strictly poorer than +// the front-loaded answer, so recover the prior turn and DISCARD the +// closer — but only when the prior turn clearly dwarfs it, because unlike a +// back-reference this closer DOES carry answer content (see modeSummary). // -// A citations addendum is tested first and wins over the back-reference test (a -// short terminal can be both), so its links are never discarded. When the -// terminal text stands on its own it is returned unchanged; when it is -// degenerate but nothing better can be recovered, it is returned as-is (a bare -// sources list still beats nothing). +// A citations addendum is tested first and wins over the other two (a short +// terminal can match more than one shape), so its links are never discarded; +// the summary closer is tested before the plain weak-final test so the +// stricter recovery bar applies when both match. When the terminal text stands +// on its own it is returned unchanged; when it is degenerate but nothing +// better can be recovered, it is returned as-is (a compressed answer still +// beats nothing). // // msgs must already include the terminal assistant message as its last element // (the loop appends it before calling this); terminal is that message's text. func finalOutput(msgs []llm.Message, terminal string) string { - citations := isCitationsOnly(terminal) - if !citations && !isWeakFinal(terminal) { + mode := modeBackRef + switch { + case isCitationsOnly(terminal): + mode = modeCitations + case isSummaryCloser(terminal): + mode = modeSummary + case !isWeakFinal(terminal): return terminal } - rec, ok := lastSubstantiveAssistantText(msgs, terminal, citations) + rec, ok := lastSubstantiveAssistantText(msgs, terminal, mode) if !ok { return terminal } - if citations { + if mode == modeCitations { // Preserve the citations addendum below the recovered answer, unless the // recovered turn already carries it (guards against a duplicate sources // block when the front-loaded turn included its own citations). The @@ -65,11 +79,47 @@ func stripURLAngles(s string) string { return strings.NewReplacer("<", "", ">", "").Replace(s) } +// recoveryMode selects the bar a prior assistant turn must clear to replace +// the terminal turn (see isSubstantiveAnswer) and what finalOutput does with +// the terminal once recovery succeeds. +type recoveryMode int + +const ( + // modeBackRef: the terminal is empty or a pure back-reference — worthless + // on its own, so any real prior answer replaces it and it is discarded. + modeBackRef recoveryMode = iota + // modeCitations: the terminal is a sources-only addendum — not a rival + // answer, so the dwarf ratio is skipped and the addendum is kept, appended + // below the recovered answer. + modeCitations + // modeSummary: the terminal acknowledges the citation round and may carry + // a short compression of the front-loaded answer. Unlike a back-reference + // it DOES contain answer content, so it is only replaced when a prior turn + // clearly dwarfs it — the ratio is mandatory at every length, and the + // closer is discarded (its content is a strict subset of what it replaced). + modeSummary +) + // backRefRe matches a terminal turn that merely points back to an earlier // message instead of stating the answer ("(Already answered above.)", // "see above", "as I said", ...). var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(said|mentioned|stated|noted)|answered (that )?above|per my (previous|earlier))`) +// summaryCloserRe matches a terminal turn that OPENS with a bookkeeping +// acknowledgment of the citation round — "Citations are logged.", "Sources +// cited.", "Logged the citations." — the shape a model produces when it +// front-loaded its answer into an earlier cite-call turn and closes by +// acknowledging the tool results, often followed by a "Short version: …" +// compression of the answer it already wrote. The ack clause must end at a +// sentence terminator ([.!]) DIRECTLY after the verb: "The citations are +// recorded in the court transcript…" is a real answer about citations, not +// bookkeeping, and must never match. A compression marker without the ack +// ("Short version: no.") is deliberately out of scope — a user who asked for +// brevity would be answered with exactly that shape, and misclassifying it +// would hijack a legitimate answer; an unmatched closer merely keeps today's +// behavior (fail closed). +var summaryCloserRe = regexp.MustCompile(`(?i)^[\s>#*_-]*((done|all set|ok(ay)?)[\s,.!:—-]+)?(((all|the)\s+)?(citations?|sources?|references?|claims?)\s+((are|were|have\s+been|all)\s+)*(logged|recorded|cited|saved|noted|captured|filed)|logged\s+((all|the)\s+)*(citations?|sources?|references?))[.!]`) + // preambleRe matches intent-announcing prefixes ("Let me search...", "I'll // check...") so a preamble is never mistaken for the answer during recovery. var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]|sure[,. ]|okay[,. ]|on it|checking)`) @@ -114,6 +164,11 @@ const ( // be at most len/N of the whole, so a prose answer that merely opens with // "Source:" and cites a URL mid-sentence is not mistaken for a bare list. citationDominatedDivisor = 3 + // summaryCloserMaxChars bounds a summary closer: room for the ack sentence + // plus a couple of compression sentences (the b3cb9ee9 closer was 151). + // Beyond this the "short version" is substantial enough that replacing it + // risks losing content the front-loaded turn never had. + summaryCloserMaxChars = 300 ) // isWeakFinal reports whether a terminal turn's text fails to stand on its own @@ -152,11 +207,26 @@ func isCitationsOnly(s string) bool { return len(residue) <= len(t)/citationDominatedDivisor } +// isSummaryCloser reports whether a terminal turn is a bookkeeping closer: it +// opens with a complete "citations are logged"-style ack sentence (see +// summaryCloserRe) and is short enough that whatever follows the ack can only +// be a compression of an earlier, fuller answer. Whether that fuller answer +// actually exists is modeSummary's job — the dwarf ratio in +// isSubstantiveAnswer keeps a matching closer in place when nothing earlier +// clearly outweighs it. +func isSummaryCloser(s string) bool { + t := strings.TrimSpace(s) + if t == "" || len(t) > summaryCloserMaxChars { + return false + } + return summaryCloserRe.MatchString(t) +} + // lastSubstantiveAssistantText scans msgs newest→oldest (skipping the terminal // turn and empty tool-only turns) for the most recent assistant turn whose text -// reads like a real answer. citations selects the recovery bar (see +// reads like a real answer. mode selects the recovery bar (see // isSubstantiveAnswer). Returns ("", false) when nothing qualifies. -func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations bool) (string, bool) { +func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, mode recoveryMode) (string, bool) { tt := strings.TrimSpace(terminal) for i := len(msgs) - 1; i >= 0; i-- { m := msgs[i] @@ -167,7 +237,7 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations if txt == "" || txt == tt { continue // the terminal turn itself, or an empty tool-only turn } - if isSubstantiveAnswer(txt, tt, citations) { + if isSubstantiveAnswer(txt, tt, mode) { return txt, true } } @@ -177,20 +247,28 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations // isSubstantiveAnswer reports whether txt (a prior assistant turn) reads like a // real answer rather than a preamble, relative to the terminal text. // -// A sufficiently long turn (>= recoverMinChars) is accepted unconditionally: a -// multi-hundred-char turn is an answer even when it opens conversationally -// ("Sure, here's…", "Let me explain: …"), so the preamble filter is NOT applied -// to it — applying it there would drop a legitimate long front-loaded answer. -// Only in the borderline band does a turn have to clear a floor, not read like a -// short planning preamble ("Let me look that up…"), and — unless the terminal is -// a citations addendum (not a rival answer, so its length is irrelevant) — also -// clearly dwarf the terminal. -func isSubstantiveAnswer(txt, terminal string, citations bool) bool { +// modeSummary demands the dwarf ratio FIRST, at every length: a summary closer +// carries a real (compressed) answer, so replacing it is only justified when +// the prior turn is clearly the fuller original it was compressed from. +// +// A sufficiently long turn (>= recoverMinChars) is otherwise accepted +// unconditionally: a multi-hundred-char turn is an answer even when it opens +// conversationally ("Sure, here's…", "Let me explain: …"), so the preamble +// filter is NOT applied to it — applying it there would drop a legitimate long +// front-loaded answer. Only in the borderline band does a turn have to clear a +// floor, not read like a short planning preamble ("Let me look that up…"), +// and — for modeBackRef only — also clearly dwarf the terminal (a citations +// addendum is not a rival answer, so its length is irrelevant; a summary +// closer already proved the ratio above). +func isSubstantiveAnswer(txt, terminal string, mode recoveryMode) bool { + if mode == modeSummary && len(txt) < recoverRatio*len(terminal) { + return false + } if len(txt) >= recoverMinChars { return true } if len(txt) < recoverFloorChars || preambleRe.MatchString(txt) { return false } - return citations || len(txt) >= recoverRatio*len(terminal) + return mode != modeBackRef || len(txt) >= recoverRatio*len(terminal) } diff --git a/agent/finalize_test.go b/agent/finalize_test.go index a8948cb..cef84ef 100644 --- a/agent/finalize_test.go +++ b/agent/finalize_test.go @@ -72,6 +72,43 @@ func TestIsCitationsOnly(t *testing.T) { } } +// b3cb9ee9Closer is the verbatim terminal turn from mort run b3cb9ee9: a +// 2,089-char answer was front-loaded into the cite-call turn and this 151-char +// compression was all that got delivered. +const b3cb9ee9Closer = "Citations are logged. Short version: the bulk of that ~$64M was AIPAC and dark-money super PACs, not the party committees — and it still wasn't enough." + +func TestIsSummaryCloser(t *testing.T) { + cases := []struct { + name string + in string + want bool + }{ + {"b3cb9ee9-verbatim", b3cb9ee9Closer, true}, + {"ack-only", "Citations are logged.", true}, + {"ack-no-copula", "Citations logged.", true}, + {"claims-cited", "All claims cited.", true}, + {"verb-first", "Logged the citations.", true}, + {"done-prefix", "Done — citations logged.", true}, + {"ack-then-tldr", "Sources have been recorded! TL;DR: the GPU was the bottleneck.", true}, + {"references-noted", "References noted. In short: yes, it ships Tuesday.", true}, + + {"empty", "", false}, + {"ack-continues-midsentence", "The citations are recorded in the court transcript, which shows the filing dates.", false}, + {"ack-verb-then-clause", "Citations are logged in Zotero whenever you click the save button.", false}, + {"compression-without-ack", "Short version: yes.", false}, // deliberately out of scope + {"mentions-citations-midsentence", "The paper's citations are what got it retracted.", false}, + {"crisp-number", "42", false}, + {"over-cap", "Citations are logged. " + strings.Repeat("The long version has many more details worth keeping. ", 6), false}, // >300: too substantial to replace + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := isSummaryCloser(c.in); got != c.want { + t.Errorf("isSummaryCloser(%q) = %v, want %v", c.in, got, c.want) + } + }) + } +} + func asst(text string, tools ...llm.ToolCall) llm.Message { m := llm.Message{Role: llm.RoleAssistant} if text != "" { @@ -83,7 +120,8 @@ func asst(text string, tools ...llm.ToolCall) llm.Message { func TestFinalOutput(t *testing.T) { cite := []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}} - longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200 + longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200 + hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12)) // >3x the b3cb9ee9 closer // A sources/citations-only terminal — the glm-5.2 "cite" shape behind mort // issue #1418: the prose answer was front-loaded into the tool-call turn and // the terminal turn carried only the citations. @@ -256,6 +294,59 @@ func TestFinalOutput(t *testing.T) { terminal: sources, want: longConversationalAnswer + "\n\n" + sources, }, + { + // The b3cb9ee9 shape: full answer front-loaded into the cite turn, + // then a summary closer. The closer is discarded — its content is a + // strict compression of the recovered answer. + name: "summary closer discarded when the front-loaded answer dwarfs it", + msgs: []llm.Message{ + llm.UserText("where did the $64M come from?"), + asst(hugeAnswer, cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst(b3cb9ee9Closer), + }, + terminal: b3cb9ee9Closer, + want: hugeAnswer, + }, + { + // The dwarf ratio is mandatory for a summary closer at EVERY length: + // a prior turn that is longer but not clearly the fuller original + // (here ~275 chars vs a 151-char closer, under the 3x bar) must not + // displace a closer that carries real answer content. + name: "summary closer kept when the prior turn does not dwarf it", + msgs: []llm.Message{ + llm.UserText("q?"), + asst(longAnswer, cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst(b3cb9ee9Closer), + }, + terminal: b3cb9ee9Closer, + want: b3cb9ee9Closer, + }, + { + // An ack-only closer ("Citations are logged.") is tiny, so even a + // modest front-loaded answer clears the ratio and replaces it. + name: "ack-only summary closer recovered over a modest answer", + msgs: []llm.Message{ + llm.UserText("q?"), + asst(longAnswer, cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst("Citations are logged."), + }, + terminal: "Citations are logged.", + want: longAnswer, + }, + { + name: "summary closer with only a preamble prior keeps the closer", + msgs: []llm.Message{ + llm.UserText("q?"), + asst("Let me gather the numbers.", cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst(b3cb9ee9Closer), + }, + terminal: b3cb9ee9Closer, + want: b3cb9ee9Closer, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -326,6 +417,37 @@ func TestRun_HealthyTerminalUnchanged(t *testing.T) { } } +// TestRun_RecoversFrontLoadedAnswerOverSummaryCloser reproduces mort run +// b3cb9ee9 end-to-end: the model front-loads its full answer into the +// cite-call turn, the cite results come back, and the terminal turn is only a +// bookkeeping ack plus a one-line compression. The delivered output must be +// the front-loaded answer, with no extra model call. +func TestRun_RecoversFrontLoadedAnswerOverSummaryCloser(t *testing.T) { + hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12)) + fp := fake.New("fp") + fp.Enqueue("test-model", + fake.ReplyWith(llm.Response{ + Parts: []llm.Part{llm.Text(hugeAnswer)}, + ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}}, + FinishReason: llm.FinishToolCalls, + Usage: llm.Usage{InputTokens: 10, OutputTokens: 5}, + }), + fake.Reply(b3cb9ee9Closer), + ) + + a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t))) + res, err := a.Run(context.Background(), "where did the $64M come from?") + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.Output != hugeAnswer { + t.Errorf("Output = %q, want recovered front-loaded answer", res.Output) + } + if n := len(fp.Calls()); n != 2 { + t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n) + } +} + // TestRun_RecoversFrontLoadedAnswerWithCitations reproduces mort issue #1418 // end-to-end: the model front-loads the prose answer into the tool-call turn // and closes with a sources-only terminal turn. The delivered output must be From 21b4775d16d51bcdbe640a9f97429f5e9341be3d Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Wed, 5 Aug 2026 20:17:56 -0400 Subject: [PATCH 2/3] =?UTF-8?q?fix(agent):=20gadfly=20round=201=20?= =?UTF-8?q?=E2=80=94=20user-boundary=20scan,=20back-ref=20precedence,=20re?= =?UTF-8?q?gex=20legibility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two behavioral fixes from the review: - modeSummary's backward scan now stops at the most recent user message. With the dwarf ratio rejecting the current turn's 1x-3x answer, the old unbounded scan could walk into WithHistory content and resurrect a stale answer to a DIFFERENT question — strictly worse than keeping the closer (opus, correctness). Other modes keep their historical unbounded scan. - A terminal matching BOTH the ack shape and a back-reference is now classified back-ref: it carries no answer content, so the looser bar is the right one (opus, error-handling). Plus the nits: summaryCloserRe assembled from named fragments, the leading marker class gains '+' (parity with citationLabelRe), verb-first form takes 'all the', dwarf ratio hoisted into one named local, and the 151-vs-153 char/byte comment inaccuracy corrected. Co-Authored-By: Claude Fable 5 --- agent/finalize.go | 66 +++++++++++++++++++++++++++++++----------- agent/finalize_test.go | 60 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 19 deletions(-) diff --git a/agent/finalize.go b/agent/finalize.go index c311b4a..88c20ac 100644 --- a/agent/finalize.go +++ b/agent/finalize.go @@ -28,15 +28,18 @@ import ( // - a bookkeeping closer ("Citations are logged. Short version: …"): the // model acknowledged the citation round and compressed the answer it had // already written into a one-liner (mort run b3cb9ee9 — a 2,089-char answer -// shrank to 151 chars at delivery). The compression is strictly poorer than -// the front-loaded answer, so recover the prior turn and DISCARD the -// closer — but only when the prior turn clearly dwarfs it, because unlike a -// back-reference this closer DOES carry answer content (see modeSummary). +// shrank to a 153-byte closer at delivery). The compression is strictly +// poorer than the front-loaded answer, so recover the prior turn and +// DISCARD the closer — but only when the prior turn clearly dwarfs it, +// because unlike a back-reference this closer DOES carry answer content +// (see modeSummary). // // A citations addendum is tested first and wins over the other two (a short -// terminal can match more than one shape), so its links are never discarded; -// the summary closer is tested before the plain weak-final test so the -// stricter recovery bar applies when both match. When the terminal text stands +// terminal can match more than one shape), so its links are never discarded. +// The back-reference test wins over the summary-closer test: a terminal +// matching both ("Citations are logged. As I said above…") carries no answer +// content of its own, so the looser back-ref recovery bar — not the summary +// closer's dwarf ratio — is the right one. When the terminal text stands // on its own it is returned unchanged; when it is degenerate but nothing // better can be recovered, it is returned as-is (a compressed answer still // beats nothing). @@ -48,9 +51,11 @@ func finalOutput(msgs []llm.Message, terminal string) string { switch { case isCitationsOnly(terminal): mode = modeCitations + case isWeakFinal(terminal): + // modeBackRef case isSummaryCloser(terminal): mode = modeSummary - case !isWeakFinal(terminal): + default: return terminal } rec, ok := lastSubstantiveAssistantText(msgs, terminal, mode) @@ -95,8 +100,11 @@ const ( // modeSummary: the terminal acknowledges the citation round and may carry // a short compression of the front-loaded answer. Unlike a back-reference // it DOES contain answer content, so it is only replaced when a prior turn - // clearly dwarfs it — the ratio is mandatory at every length, and the - // closer is discarded (its content is a strict subset of what it replaced). + // clearly dwarfs it — the ratio is mandatory at every length, the recovery + // scan stops at the most recent user message (a compression can only be of + // THIS turn's answer; never resurrect one from an earlier question), and + // the closer is discarded (its content is a strict subset of what it + // replaced). modeSummary ) @@ -117,8 +125,20 @@ var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(sai // ("Short version: no.") is deliberately out of scope — a user who asked for // brevity would be answered with exactly that shape, and misclassifying it // would hijack a legitimate answer; an unmatched closer merely keeps today's -// behavior (fail closed). -var summaryCloserRe = regexp.MustCompile(`(?i)^[\s>#*_-]*((done|all set|ok(ay)?)[\s,.!:—-]+)?(((all|the)\s+)?(citations?|sources?|references?|claims?)\s+((are|were|have\s+been|all)\s+)*(logged|recorded|cited|saved|noted|captured|filed)|logged\s+((all|the)\s+)*(citations?|sources?|references?))[.!]`) +// behavior (fail closed). Assembled from named fragments so the alternations +// stay legible and extendable. +const ( + summaryLead = `[\s>#*_+-]*` // leading markdown/list markers, as in citationLabelRe + summaryPreface = `((done|all set|ok(ay)?)[\s,.!:—-]+)?` // optional "Done —" style opener + summaryNouns = `(citations?|sources?|references?|claims?)` + summaryCopulas = `((are|were|have\s+been|all)\s+)*` + summaryVerbs = `(logged|recorded|cited|saved|noted|captured|filed)` + summaryArticle = `((all|the)\s+)*` // star, not ?: "Logged all the citations." +) + +var summaryCloserRe = regexp.MustCompile(`(?i)^` + summaryLead + summaryPreface + + `(` + summaryArticle + summaryNouns + `\s+` + summaryCopulas + summaryVerbs + + `|logged\s+` + summaryArticle + summaryNouns + `)[.!]`) // preambleRe matches intent-announcing prefixes ("Let me search...", "I'll // check...") so a preamble is never mistaken for the answer during recovery. @@ -165,9 +185,10 @@ const ( // "Source:" and cites a URL mid-sentence is not mistaken for a bare list. citationDominatedDivisor = 3 // summaryCloserMaxChars bounds a summary closer: room for the ack sentence - // plus a couple of compression sentences (the b3cb9ee9 closer was 151). - // Beyond this the "short version" is substantial enough that replacing it - // risks losing content the front-loaded turn never had. + // plus a couple of compression sentences (the b3cb9ee9 closer was 153 + // bytes — Go len(), which is what every threshold here compares). Beyond + // this the "short version" is substantial enough that replacing it risks + // losing content the front-loaded turn never had. summaryCloserMaxChars = 300 ) @@ -230,6 +251,16 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, mode reco tt := strings.TrimSpace(terminal) for i := len(msgs) - 1; i >= 0; i-- { m := msgs[i] + if mode == modeSummary && m.Role == llm.RoleUser { + // A summary closer compresses THIS turn's front-loaded answer, so + // the scan must not cross into an earlier question: once the dwarf + // ratio has rejected the current turn's text, walking further back + // would resurrect a stale answer to a DIFFERENT question — strictly + // worse than keeping the closer. (A mid-run steer message is also a + // user-role boundary; recovery then fails closed, which is fine.) + // The other modes keep their historical unbounded scan. + break + } if m.Role != llm.RoleAssistant { continue } @@ -261,7 +292,8 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, mode reco // addendum is not a rival answer, so its length is irrelevant; a summary // closer already proved the ratio above). func isSubstantiveAnswer(txt, terminal string, mode recoveryMode) bool { - if mode == modeSummary && len(txt) < recoverRatio*len(terminal) { + dwarfs := len(txt) >= recoverRatio*len(terminal) + if mode == modeSummary && !dwarfs { return false } if len(txt) >= recoverMinChars { @@ -270,5 +302,5 @@ func isSubstantiveAnswer(txt, terminal string, mode recoveryMode) bool { if len(txt) < recoverFloorChars || preambleRe.MatchString(txt) { return false } - return mode != modeBackRef || len(txt) >= recoverRatio*len(terminal) + return mode != modeBackRef || dwarfs } diff --git a/agent/finalize_test.go b/agent/finalize_test.go index cef84ef..89703fb 100644 --- a/agent/finalize_test.go +++ b/agent/finalize_test.go @@ -73,8 +73,9 @@ func TestIsCitationsOnly(t *testing.T) { } // b3cb9ee9Closer is the verbatim terminal turn from mort run b3cb9ee9: a -// 2,089-char answer was front-loaded into the cite-call turn and this 151-char -// compression was all that got delivered. +// 2,089-char answer was front-loaded into the cite-call turn and this 153-byte +// compression (151 runes — the em dash is 3 bytes, and byte length is what the +// thresholds compare) was all that got delivered. const b3cb9ee9Closer = "Citations are logged. Short version: the bulk of that ~$64M was AIPAC and dark-money super PACs, not the party committees — and it still wasn't enough." func TestIsSummaryCloser(t *testing.T) { @@ -91,6 +92,8 @@ func TestIsSummaryCloser(t *testing.T) { {"done-prefix", "Done — citations logged.", true}, {"ack-then-tldr", "Sources have been recorded! TL;DR: the GPU was the bottleneck.", true}, {"references-noted", "References noted. In short: yes, it ships Tuesday.", true}, + {"plus-list-marker", "+ Citations are logged.", true}, + {"logged-all-the", "Logged all the citations.", true}, {"empty", "", false}, {"ack-continues-midsentence", "The citations are recorded in the court transcript, which shows the filing dates.", false}, @@ -141,6 +144,9 @@ func TestFinalOutput(t *testing.T) { // A >=200-byte real answer that merely OPENS with a conversational word // ("Sure,"). The preamble filter must NOT veto it (gadfly regression guard). longConversationalAnswer := "Sure, here's the rundown: it currently sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle, with the sealed Founders Edition commanding the top of that range while used AIB cards go a bit lower." + // Matches BOTH the summary ack and backRefRe, within the 120-byte weak cap, + // and long enough (>~92 bytes) that longAnswer would fail the summary bar. + bothMatchCloser := "Citations are logged. As I mentioned above, the full detail on the money sources is in my earlier message." tests := []struct { name string @@ -347,6 +353,56 @@ func TestFinalOutput(t *testing.T) { terminal: b3cb9ee9Closer, want: b3cb9ee9Closer, }, + { + // Gadfly (opus, correctness): the modeSummary scan must stop at the + // most recent user message. Here the current turn's answer sits in + // the 1x-3x band (rejected by the ratio) while a dwarfing answer to + // a DIFFERENT question sits in history — resurrecting it would be + // strictly worse than keeping the closer. + name: "summary closer never resurrects a stale answer across the user boundary", + msgs: []llm.Message{ + llm.UserText("earlier, unrelated question?"), + asst(hugeAnswer), + llm.UserText("q?"), + asst(conciseAnswer, cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst(b3cb9ee9Closer), + }, + terminal: b3cb9ee9Closer, + want: b3cb9ee9Closer, + }, + { + // The boundary must not break the legitimate multi-turn case: the + // dwarfing front-loaded answer in THIS turn's window is recovered + // even with history behind it. + name: "summary closer recovery still works with history present", + msgs: []llm.Message{ + llm.UserText("earlier, unrelated question?"), + asst(longAnswer), + llm.UserText("q?"), + asst(hugeAnswer, cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst(b3cb9ee9Closer), + }, + terminal: b3cb9ee9Closer, + want: hugeAnswer, + }, + { + // Gadfly (opus, error-handling): a closer matching BOTH the ack + // shape and a back-reference carries no answer content, so the + // back-ref test must win and the ordinary recovery bar apply — + // under the summary bar this ~106-byte terminal would demand a + // ~318-byte prior and wrongly keep the closer over longAnswer. + name: "back-reference wins over the summary ack when both match", + msgs: []llm.Message{ + llm.UserText("q?"), + asst(longAnswer, cite...), + llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}), + asst(bothMatchCloser), + }, + terminal: bothMatchCloser, + want: longAnswer, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { From 1bbbdaa1e5b5170789bad05c68f89345b5d0ec4f Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Wed, 5 Aug 2026 20:29:19 -0400 Subject: [PATCH 3/3] =?UTF-8?q?refactor(agent):=20gadfly=20round=202=20?= =?UTF-8?q?=E2=80=94=20shared=20leadingMarkers,=20explicit=20mode,=20comme?= =?UTF-8?q?nt=20altitude?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All tidiness, no behavior change: the leading-marker class is one shared constant for citationLabelRe and summaryCloserRe (hand-copying it is how '+' went missing the first time); the deliberate 'all' duplication across summaryCopulas/summaryArticle is now stated at both sites; the weak-final switch case assigns modeBackRef explicitly; test comments state the constraint they guard instead of which reviewer asked for them. Co-Authored-By: Claude Fable 5 --- agent/finalize.go | 18 ++++++++++++++---- agent/finalize_test.go | 20 ++++++++++---------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/agent/finalize.go b/agent/finalize.go index 88c20ac..72b9e77 100644 --- a/agent/finalize.go +++ b/agent/finalize.go @@ -52,7 +52,7 @@ func finalOutput(msgs []llm.Message, terminal string) string { case isCitationsOnly(terminal): mode = modeCitations case isWeakFinal(terminal): - // modeBackRef + mode = modeBackRef case isSummaryCloser(terminal): mode = modeSummary default: @@ -128,15 +128,17 @@ var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(sai // behavior (fail closed). Assembled from named fragments so the alternations // stay legible and extendable. const ( - summaryLead = `[\s>#*_+-]*` // leading markdown/list markers, as in citationLabelRe summaryPreface = `((done|all set|ok(ay)?)[\s,.!:—-]+)?` // optional "Done —" style opener summaryNouns = `(citations?|sources?|references?|claims?)` + // "all" appears here AND in summaryArticle on purpose: as a quantifier + // between noun and verb ("Citations all logged.") and as a determiner + // before the noun ("All claims cited.", "Logged all the citations."). summaryCopulas = `((are|were|have\s+been|all)\s+)*` summaryVerbs = `(logged|recorded|cited|saved|noted|captured|filed)` summaryArticle = `((all|the)\s+)*` // star, not ?: "Logged all the citations." ) -var summaryCloserRe = regexp.MustCompile(`(?i)^` + summaryLead + summaryPreface + +var summaryCloserRe = regexp.MustCompile(`(?i)^` + leadingMarkers + summaryPreface + `(` + summaryArticle + summaryNouns + `\s+` + summaryCopulas + summaryVerbs + `|logged\s+` + summaryArticle + summaryNouns + `)[.!]`) @@ -153,7 +155,15 @@ var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]| // the colon/dash separator. Anchored at ^ so a normal answer that merely // mentions "sources" mid-sentence, or ends with a "Sources:" section AFTER its // prose, is never matched. -var citationLabelRe = regexp.MustCompile(`(?i)^[\s>#*_+-]*(sources?|references?|citations?|works cited|further reading)\b[\s*_]*[:：\-—]`) +var citationLabelRe = regexp.MustCompile(`(?i)^` + leadingMarkers + + `(sources?|references?|citations?|works cited|further reading)\b[\s*_]*[:：\-—]`) + +// leadingMarkers tolerates markdown noise before a label: emphasis (*, _), +// list (-, +, *), block-quote (>), and ATX-heading (#) markers, with their +// whitespace. Shared by citationLabelRe and summaryCloserRe so the two +// classifiers cannot drift apart (the first draft of the summary class +// dropped '+' by hand-copying this set). +const leadingMarkers = `[\s>#*_+-]*` // linkRe matches a whole markdown link "[label](url)" or a bare URL. Used both // to require that a citations terminal carries at least one link and to strip diff --git a/agent/finalize_test.go b/agent/finalize_test.go index 89703fb..61e6586 100644 --- a/agent/finalize_test.go +++ b/agent/finalize_test.go @@ -354,11 +354,11 @@ func TestFinalOutput(t *testing.T) { want: b3cb9ee9Closer, }, { - // Gadfly (opus, correctness): the modeSummary scan must stop at the - // most recent user message. Here the current turn's answer sits in - // the 1x-3x band (rejected by the ratio) while a dwarfing answer to - // a DIFFERENT question sits in history — resurrecting it would be - // strictly worse than keeping the closer. + // The modeSummary scan must stop at the most recent user message. + // Here the current turn's answer sits in the 1x-3x band (rejected + // by the ratio) while a dwarfing answer to a DIFFERENT question + // sits in history — resurrecting it would be strictly worse than + // keeping the closer. name: "summary closer never resurrects a stale answer across the user boundary", msgs: []llm.Message{ llm.UserText("earlier, unrelated question?"), @@ -388,11 +388,11 @@ func TestFinalOutput(t *testing.T) { want: hugeAnswer, }, { - // Gadfly (opus, error-handling): a closer matching BOTH the ack - // shape and a back-reference carries no answer content, so the - // back-ref test must win and the ordinary recovery bar apply — - // under the summary bar this ~106-byte terminal would demand a - // ~318-byte prior and wrongly keep the closer over longAnswer. + // A closer matching BOTH the ack shape and a back-reference + // carries no answer content, so the back-ref test must win and the + // ordinary recovery bar apply — under the summary bar this + // ~106-byte terminal would demand a ~318-byte prior and wrongly + // keep the closer over longAnswer. name: "back-reference wins over the summary ack when both match", msgs: []llm.Message{ llm.UserText("q?"),