fix(agent): recover front-loaded answer past a citations-only terminal turn (#11)

finalOutput now recovers the front-loaded answer when the terminal turn is a sources/citations-only addendum ("Sources: [x](url), ..."), not just when it is empty or a back-reference. It recovers the prior substantive answer and appends the (real) citations below it. Guards: citation-DOMINANCE (a prose answer that merely opens with "Source: ... http://..." is left as the answer), ^-anchored heading, citations recovery decoupled from the terminal-length ratio (concise answers recover too), preamble filter applied only in the borderline band (long answers opening with "Sure,"/"Let me" are not vetoed), and a dedup that ignores <url> angle-bracket wrappers. Healthy terminal answers unchanged; zero extra model calls. Fixes mort #1418. Gadfly-reviewed (6 reviewers) + adversarially pre-verified; all findings graded, real ones addressed. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-10 16:47:32 +00:00
parent fe44a6da26
commit 6abb399f5b
2 changed files with 295 additions and 18 deletions
@@ -14,21 +14,55 @@ import (
 // their answer to the final, tool-free turn. But some models — notably several
 // open-weight ones — "front-load" their full answer into an earlier turn that
 // ALSO calls a tool (e.g. answer text alongside a citation call), then close
-// with a trivial pointer such as "(Already answered above.)". Returning only
-// the terminal text would discard the real answer, which is still present
-// earlier in the transcript. When the terminal text is weak (empty, or a short
-// back-reference) fall back to the last substantive assistant content in msgs.
+// with a degenerate terminal turn that is not itself the answer. Two shapes are
+// recovered from the transcript (zero extra model calls):
+//
+//   - a trivial back-reference ("(Already answered above.)", "see above", …):
+//     the real answer sits earlier, so recover it and DISCARD the worthless
+//     closer.
+//   - a sources/citations-only addendum ("Sources: [x](…), [y](…)"): the model
+//     front-loaded the prose answer and closed with just its citations (the
+//     glm-5.2 "cite" pattern behind mort issue #1418). The citations are real,
+//     useful content — unlike a back-reference — so recover the prior answer and
+//     KEEP the citations, appended below it.
+//
+// A citations addendum is tested first and wins over the back-reference test (a
+// short terminal can be both), so its links are never discarded. When the
+// terminal text stands on its own it is returned unchanged; when it is
+// degenerate but nothing better can be recovered, it is returned as-is (a bare
+// sources list still beats nothing).
 //
 // msgs must already include the terminal assistant message as its last element
 // (the loop appends it before calling this); terminal is that message's text.
 func finalOutput(msgs []llm.Message, terminal string) string {
-	if !isWeakFinal(terminal) {
+	citations := isCitationsOnly(terminal)
+	if !citations && !isWeakFinal(terminal) {
 		return terminal
 	}
-	if rec, ok := lastSubstantiveAssistantText(msgs, terminal); ok {
-		return rec
+	rec, ok := lastSubstantiveAssistantText(msgs, terminal, citations)
+	if !ok {
+		return terminal
 	}
-	return terminal
+	if citations {
+		// Preserve the citations addendum below the recovered answer, unless the
+		// recovered turn already carries it (guards against a duplicate sources
+		// block when the front-loaded turn included its own citations). The
+		// containment test ignores <url> angle-bracket wrappers so a turn that
+		// listed the same sources unwrapped still suppresses the duplicate.
+		if tail := strings.TrimSpace(terminal); !strings.Contains(stripURLAngles(rec), stripURLAngles(tail)) {
+			return rec + "\n\n" + tail
+		}
+	}
+	return rec
+}
+
+// stripURLAngles removes the <…> wrappers Discord uses to suppress link embeds,
+// so the citations dedup compares URLs regardless of that formatting delta.
+func stripURLAngles(s string) string {
+	if !strings.ContainsAny(s, "<>") {
+		return s
+	}
+	return strings.NewReplacer("<", "", ">", "").Replace(s)
 }

 // backRefRe matches a terminal turn that merely points back to an earlier
@@ -40,19 +74,46 @@ var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(sai
 // check...") so a preamble is never mistaken for the answer during recovery.
 var preambleRe = regexp.MustCompile(`(?i)^(let me|let'?s|i'?ll|i will|first[, ]|sure[,. ]|okay[,. ]|on it|checking)`)

+// citationLabelRe matches a terminal turn that OPENS with a sources/citations
+// heading — the shape a model produces when it front-loads its answer into an
+// earlier tool-call turn and closes with only its sources. Leading markdown
+// emphasis (*, _), list (-, +, *), block-quote (>), and ATX-heading (#) markers
+// — with their whitespace, since \s is in the class — are tolerated before the
+// label, as is closing emphasis (** / __) plus whitespace between the label and
+// the colon/dash separator. Anchored at ^ so a normal answer that merely
+// mentions "sources" mid-sentence, or ends with a "Sources:" section AFTER its
+// prose, is never matched.
+var citationLabelRe = regexp.MustCompile(`(?i)^[\s>#*_+-]*(sources?|references?|citations?|works cited|further reading)\b[\s*_]*[:：\-—]`)
+
+// linkRe matches a whole markdown link "[label](url)" or a bare URL. Used both
+// to require that a citations terminal carries at least one link and to strip
+// links out when measuring how citation-dominated the terminal is.
+var linkRe = regexp.MustCompile(`\[[^\]]*\]\([^)]*\)|https?://\S+`)
+
+// citationResidueCutset is trimmed from the ends of a citations terminal's
+// non-link remainder before measuring it — list bullets, separators, and the
+// short per-source annotations models add in parentheses.
+const citationResidueCutset = " \t\r\n,;.:|·•*_()[]—-"
+
 const (
 	// weakFinalMaxChars bounds how long a back-reference closer can be. A
 	// genuine final answer that merely contains "as I said" mid-sentence is
 	// longer than this, so it is never treated as weak.
 	weakFinalMaxChars = 120
 	// recoverMinChars: a prior assistant turn this long is treated as a real
-	// answer regardless of how it opens.
+	// answer regardless of how it opens (the preamble filter is not applied at
+	// this length — see isSubstantiveAnswer).
 	recoverMinChars = 200
 	// recoverFloorChars / recoverRatio gate the borderline band: a shorter
-	// prior turn must still clearly dwarf the (very short) terminal and not
-	// look like a preamble.
+	// prior turn must clear the floor and — unless the terminal is a citations
+	// addendum, which is not a rival answer — also clearly dwarf the (very
+	// short) terminal. See isSubstantiveAnswer.
 	recoverFloorChars = 80
 	recoverRatio      = 3
+	// citationDominatedDivisor: a citations terminal's non-link remainder must
+	// be at most len/N of the whole, so a prose answer that merely opens with
+	// "Source:" and cites a URL mid-sentence is not mistaken for a bare list.
+	citationDominatedDivisor = 3
 )

 // isWeakFinal reports whether a terminal turn's text fails to stand on its own
@@ -65,10 +126,37 @@ func isWeakFinal(s string) bool {
 	return len(t) <= weakFinalMaxChars && backRefRe.MatchString(t)
 }

+// isCitationsOnly reports whether a terminal turn is essentially just a
+// sources/citations addendum: it OPENS with a citations heading, carries at
+// least one link, and — once the heading and links are removed — is dominated
+// by that citation structure (only list punctuation and short per-source
+// annotations remain). The dominance check is what separates a bare sources
+// list (recover the front-loaded answer, keep the links) from a real prose
+// answer that merely opens with "Source:" and references a URL mid-sentence
+// (leave it as the answer). Unlike a back-reference closer the links are worth
+// keeping, so finalOutput appends them to the recovered answer.
+//
+// A citations terminal whose sources are bare domains (no scheme, no markdown
+// link) is intentionally out of scope — there is no reliable link signal, so it
+// is left as-is rather than risk misclassifying prose.
+func isCitationsOnly(s string) bool {
+	t := strings.TrimSpace(s)
+	if !citationLabelRe.MatchString(t) {
+		return false
+	}
+	body := citationLabelRe.ReplaceAllString(t, "")
+	if !linkRe.MatchString(body) {
+		return false
+	}
+	residue := strings.Trim(linkRe.ReplaceAllString(body, ""), citationResidueCutset)
+	return len(residue) <= len(t)/citationDominatedDivisor
+}
+
 // lastSubstantiveAssistantText scans msgs newest→oldest (skipping the terminal
 // turn and empty tool-only turns) for the most recent assistant turn whose text
-// reads like a real answer. Returns ("", false) when nothing qualifies.
-func lastSubstantiveAssistantText(msgs []llm.Message, terminal string) (string, bool) {
+// reads like a real answer. citations selects the recovery bar (see
+// isSubstantiveAnswer). Returns ("", false) when nothing qualifies.
+func lastSubstantiveAssistantText(msgs []llm.Message, terminal string, citations bool) (string, bool) {
 	tt := strings.TrimSpace(terminal)
 	for i := len(msgs) - 1; i >= 0; i-- {
 		m := msgs[i]
@@ -79,7 +167,7 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string) (string,
 		if txt == "" || txt == tt {
 			continue // the terminal turn itself, or an empty tool-only turn
 		}
-		if isSubstantiveAnswer(txt, tt) {
+		if isSubstantiveAnswer(txt, tt, citations) {
 			return txt, true
 		}
 	}
@@ -88,11 +176,21 @@ func lastSubstantiveAssistantText(msgs []llm.Message, terminal string) (string,

 // isSubstantiveAnswer reports whether txt (a prior assistant turn) reads like a
 // real answer rather than a preamble, relative to the terminal text.
-func isSubstantiveAnswer(txt, terminal string) bool {
+//
+// A sufficiently long turn (>= recoverMinChars) is accepted unconditionally: a
+// multi-hundred-char turn is an answer even when it opens conversationally
+// ("Sure, here's…", "Let me explain: …"), so the preamble filter is NOT applied
+// to it — applying it there would drop a legitimate long front-loaded answer.
+// Only in the borderline band does a turn have to clear a floor, not read like a
+// short planning preamble ("Let me look that up…"), and — unless the terminal is
+// a citations addendum (not a rival answer, so its length is irrelevant) — also
+// clearly dwarf the terminal.
+func isSubstantiveAnswer(txt, terminal string, citations bool) bool {
 	if len(txt) >= recoverMinChars {
 		return true
 	}
-	return len(txt) >= recoverFloorChars &&
-		len(txt) >= recoverRatio*len(terminal) &&
-		!preambleRe.MatchString(txt)
+	if len(txt) < recoverFloorChars || preambleRe.MatchString(txt) {
+		return false
+	}
+	return citations || len(txt) >= recoverRatio*len(terminal)
 }
@@ -36,6 +36,42 @@ func TestIsWeakFinal(t *testing.T) {
 	}
 }

+func TestIsCitationsOnly(t *testing.T) {
+	cases := []struct {
+		name string
+		in   string
+		want bool
+	}{
+		{"sources-md-links", "Sources: [pcprice.watch](https://pcprice.watch/x), [ebay](https://ebay.com/1).", true},
+		{"lowercase-bare-url", "sources: see https://example.com/a", true},
+		{"bold-label-colon-inside", "**Sources:** [a](https://a), [b](https://b)", true},
+		{"bold-label-colon-outside", "**Sources**: [a](https://a), [b](https://b)", true}, // colon after the closing **
+		{"references-dash", "References — [a](https://a)", true},
+		{"citations-label", "Citations: https://x/y", true},
+		{"leading-list-marker", "- Sources: [a](https://a)", true},
+		{"atx-heading", "## Sources: [a](https://a), [b](https://b)", true}, // ATX heading marker + its trailing space
+		{"further-reading", "Further reading: https://example.com/deep-dive", true},
+		{"annotated-multi-source", "Sources: [pcprice.watch](https://a) (tracker), [eBay](https://b) (sold), [bestvaluegpu](https://c) (retail), [resaleprices](https://d) (asking).", true}, // the reported issue-1418 shape
+		{"backref-plus-links-is-citations", "References: as noted above, [pcprice.watch](https://pcprice.watch/x).", true},                                                                   // a back-ref phrase inside a real sources list is still citations
+
+		{"empty", "", false},
+		{"label-but-no-link", "Source: internal analysis, no URL here", false},
+		{"prose-then-sources", "It sells for ~$2,700. Sources: [a](https://a)", false}, // answer first → not a pure addendum
+		{"source-led-prose-answer", "Source: According to https://cdc.gov the flu vaccine is 40-60% effective, and the CDC recommends annual vaccination for everyone over six months old.", false}, // a prose answer that merely opens with a "Source:" label
+		{"mentions-sources-midsentence", "The sources of the leak were never confirmed.", false},
+		{"link-without-label", "Here is the link you asked for: [a](https://a)", false},
+		{"bare-domains-out-of-scope", "Sources: pcprice.watch (used ~$200), ebay.com (sold listings)", false}, // bare domains: no scheme or markdown link to key on
+		{"crisp-number", "42", false},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if got := isCitationsOnly(c.in); got != c.want {
+				t.Errorf("isCitationsOnly(%q) = %v, want %v", c.in, got, c.want)
+			}
+		})
+	}
+}
+
 func asst(text string, tools ...llm.ToolCall) llm.Message {
 	m := llm.Message{Role: llm.RoleAssistant}
 	if text != "" {
@@ -48,6 +84,25 @@ func asst(text string, tools ...llm.ToolCall) llm.Message {
 func TestFinalOutput(t *testing.T) {
 	cite := []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}}
 	longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200
+	// A sources/citations-only terminal — the glm-5.2 "cite" shape behind mort
+	// issue #1418: the prose answer was front-loaded into the tool-call turn and
+	// the terminal turn carried only the citations.
+	sources := "Sources: [pcprice.watch](https://pcprice.watch/x), [ebay](https://www.ebay.com/itm/1)."
+	answerWithSources := longAnswer + "\n\n" + sources
+	// A concise (>80, <200 byte) front-loaded answer + a long citations terminal:
+	// the ratio arm can't be met against the long terminal, so citations mode
+	// must fall back to the floor.
+	conciseAnswer := "It sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle."
+	longSources := "Sources: [pcprice.watch](https://pcprice.watch/gpu/rtx5090) (tracker), [ebay](https://www.ebay.com/sch/rtx5090) (sold), [newegg](https://newegg.com/rtx5090) (retail), [pcpartpicker](https://pcpartpicker.com/rtx5090) (history)."
+	// A substantive answer that merely OPENS with "Source:" (not a bare list).
+	sourceLedAnswer := "Source: https://nvd.nist.gov/vuln/detail/CVE-2024-1234 — this is the authoritative NVD entry for the vulnerability, rated CVSS 9.8 critical."
+	// A borderline-band (80–200 byte) turn that opens like a planning preamble:
+	// it clears the floor, but the preamble filter still vetoes it (the filter
+	// applies only in the borderline band; a >=200-byte turn is accepted as-is).
+	preambleTurn := "Let me look that up across a few different sites and then compile the full comparison for you here."
+	// A >=200-byte real answer that merely OPENS with a conversational word
+	// ("Sure,"). The preamble filter must NOT veto it (gadfly regression guard).
+	longConversationalAnswer := "Sure, here's the rundown: it currently sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle, with the sealed Founders Edition commanding the top of that range while used AIB cards go a bit lower."

 	tests := []struct {
 		name     string
@@ -110,6 +165,97 @@ func TestFinalOutput(t *testing.T) {
 			terminal: "(see above)",
 			want:     "(see above)", // preamble excluded; falls back to terminal
 		},
+		{
+			name: "citations-only terminal recovers front-loaded answer and keeps sources",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst(longAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(sources),
+			},
+			terminal: sources,
+			want:     answerWithSources, // answer recovered, citations appended
+		},
+		{
+			name: "citations-only terminal but only a preamble prior: keeps the sources",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst("Let me gather the sources.", cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(sources),
+			},
+			terminal: sources,
+			want:     sources, // nothing substantive to recover → keep the addendum
+		},
+		{
+			name: "citations already in the recovered answer are not duplicated",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst(answerWithSources, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(sources),
+			},
+			terminal: sources,
+			want:     answerWithSources, // recovered turn already carries the sources
+		},
+		{
+			// #1418 persisted for CONCISE answers: a <200-char front-loaded
+			// answer must still be recovered against a long citations terminal
+			// (the ratio arm is skipped in citations mode).
+			name: "concise front-loaded answer recovered against a long citations terminal",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst(conciseAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(longSources),
+			},
+			terminal: longSources,
+			want:     conciseAnswer + "\n\n" + longSources,
+		},
+		{
+			// A substantive answer that merely OPENS with "Source:" and cites a
+			// URL mid-sentence is NOT a citations addendum — return it verbatim,
+			// never prepend the prior planning turn.
+			name: "source-led substantive answer is not hijacked by a prior turn",
+			msgs: []llm.Message{
+				llm.UserText("what's the authoritative URL?"),
+				asst("I'll look up the CVE in the NVD database, cross-reference the vendor advisory, and confirm the canonical URL before I answer.", cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(sourceLedAnswer),
+			},
+			terminal: sourceLedAnswer,
+			want:     sourceLedAnswer,
+		},
+		{
+			// A borderline-length turn that opens like a preamble is vetoed
+			// during recovery; the older real answer is recovered instead. (A
+			// >=200-byte turn would be accepted verbatim — see the next case.)
+			name: "borderline preamble is skipped; older real answer recovered",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst(conciseAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(preambleTurn, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c2", Name: "cite", Content: "ok"}),
+				asst(sources),
+			},
+			terminal: sources,
+			want:     conciseAnswer + "\n\n" + sources,
+		},
+		{
+			// Guards the gadfly regression: a LONG (>=200-byte) front-loaded
+			// answer that merely opens with a conversational word ("Sure, …")
+			// must still be recovered — the preamble filter must not veto it.
+			name: "long answer opening with a conversational word is still recovered",
+			msgs: []llm.Message{
+				llm.UserText("q?"),
+				asst(longConversationalAnswer, cite...),
+				llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
+				asst(sources),
+			},
+			terminal: sources,
+			want:     longConversationalAnswer + "\n\n" + sources,
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
@@ -179,3 +325,36 @@ func TestRun_HealthyTerminalUnchanged(t *testing.T) {
 		t.Errorf("Output = %q, want terminal answer unchanged", res.Output)
 	}
 }
+
+// TestRun_RecoversFrontLoadedAnswerWithCitations reproduces mort issue #1418
+// end-to-end: the model front-loads the prose answer into the tool-call turn
+// and closes with a sources-only terminal turn. The delivered output must be
+// the recovered answer with the citations appended (not the bare sources list),
+// with no extra model call.
+func TestRun_RecoversFrontLoadedAnswerWithCitations(t *testing.T) {
+	longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6))
+	sources := "Sources: [docs](https://example.com/docs), [pricing](https://example.com/pricing)."
+	fp := fake.New("fp")
+	fp.Enqueue("test-model",
+		fake.ReplyWith(llm.Response{
+			Parts:        []llm.Part{llm.Text(longAnswer)},
+			ToolCalls:    []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
+			FinishReason: llm.FinishToolCalls,
+			Usage:        llm.Usage{InputTokens: 10, OutputTokens: 5},
+		}),
+		fake.Reply(sources),
+	)
+
+	a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
+	res, err := a.Run(context.Background(), "is there a meet time limit?")
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	want := longAnswer + "\n\n" + sources
+	if res.Output != want {
+		t.Errorf("Output = %q, want recovered answer + citations %q", res.Output, want)
+	}
+	if n := len(fp.Calls()); n != 2 {
+		t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
+	}
+}