fix(agent): recover the answer behind a bare "above" pointer
mort issue #1611: run 8eea3e82 front-loaded a 2,245-char analysis into its cite-call turn and closed with 220 bytes — "Done — that's the full chain above. Short version: …". The user got the 220 bytes and a pointer at a chain that was never posted. finalOutput already had three shapes for this pathology, and the closer matched none of them: too long for the weak-final cap (220 > 120), no citations heading, and no "Citations are logged." ack to open the summary-closer class. So it was delivered verbatim. The three shapes were each a separate vocabulary of ack phrases, which is why a fourth phrasing walked straight through. Two of them are really one signal — the terminal DEFERS, telling us the answer is somewhere the user cannot see — differing only in whether the terminal also carries content of its own. That signal is now isBackRef, shared by both, so a new phrasing is added once and covered in the bare and the "+ compression" variant at the same time. Its open-ended half is aboveRefRe: a DEICTIC "above", separated from the preposition by what follows the word. The deictic use ends its clause ("that's the full chain above.", "as shown above,"); the preposition always continues into a noun phrase ("above 100°C", "above the fold", "above all, …"). pointsAbove additionally requires the reference in the terminal's first 120 bytes — with almost no text before it in THIS message, it cannot be pointing at the message's own content. isSummaryCloser now opens on either the citations ack or pointsAbove. Recovery is unchanged: the mandatory dwarf ratio and the user-message scan boundary still gate it, so a closer only loses to a prior turn in the same user turn that is clearly the fuller original. Break-checked: reverting either classifier, dropping the clause-final rule, or dropping the offset bound each kills a named test; the unmutated control survives.
This commit is contained in:
@@ -26,6 +26,12 @@ func TestIsWeakFinal(t *testing.T) {
|
||||
{"crisp-yes", "Yes.", false},
|
||||
{"crisp-status", "It's down, restarting now.", false},
|
||||
{"long-with-as-i-said", long, false}, // >120 chars: not weak despite the phrase
|
||||
|
||||
// The deictic half of the class (aboveRefRe), inside the length cap.
|
||||
{"bare-above-pointer", "That's the full chain above.", true},
|
||||
{"above-pointer-in-parens", "(the breakdown is above)", true},
|
||||
{"prepositional-above-not-weak", "Anything above 100 degrees boils off.", false},
|
||||
{"1611-closer-too-long-for-weak", closer1611, false}, // 220 bytes: isSummaryCloser's job
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
@@ -36,6 +42,53 @@ func TestIsWeakFinal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// closer1611 is the verbatim terminal turn from mort run
|
||||
// 8eea3e82-b61b-4174-9750-9aa2f4bde4d4 (issue #1611): the model front-loaded a
|
||||
// 2,245-char analysis into the cite-call turn and closed with this 220-byte
|
||||
// pointer-plus-compression. Too long for the weak-final cap and carrying no
|
||||
// citations ack, it matched none of the three original shapes and was
|
||||
// delivered verbatim — the user saw a summary referring to a "chain above"
|
||||
// that had never been posted.
|
||||
const closer1611 = "Done — that's the full chain above. Short version: it's not one incident, it's the confluence of the Iran war, the Epstein files, and three ex-allies now openly plotting a third-party movement that finally set him off."
|
||||
|
||||
func TestPointsAbove(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
// Deictic: "above" ends its clause, so it points at earlier text.
|
||||
{"1611-verbatim", closer1611, true},
|
||||
{"sentence-final", "That's the full chain above.", true},
|
||||
{"comma", "As shown above, the answer is 60 minutes.", true},
|
||||
{"end-of-string", "The full breakdown is above", true},
|
||||
{"line-final", "Everything is above\n\nShort version: yes.", true},
|
||||
{"closing-paren", "(the detail is above).", true},
|
||||
{"semicolon", "It's above; the short answer is no.", true},
|
||||
|
||||
// Prepositional: "above" continues into a noun phrase.
|
||||
{"above-a-number", "Anything above 100 degrees boils off.", false},
|
||||
{"above-the-fold", "The banner sits above the fold on every page.", false},
|
||||
{"above-all", "Above all, keep the deploy green.", false},
|
||||
{"above-average", "Turnout was above average in three counties.", false},
|
||||
|
||||
{"no-above-at-all", "42", false},
|
||||
{"empty", "", false},
|
||||
|
||||
// Offset bound: past backRefHeadChars there IS enough text before the
|
||||
// reference for it to be pointing inside this same message.
|
||||
{"late-reference-not-a-pointer", strings.Repeat("x", backRefHeadChars+1) + " as shown above.", false},
|
||||
{"reference-at-the-bound", strings.Repeat("x", backRefHeadChars-6) + " above.", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := pointsAbove(c.in); got != c.want {
|
||||
t.Errorf("pointsAbove(%q) = %v, want %v", c.in, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsCitationsOnly(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -102,6 +155,13 @@ func TestIsSummaryCloser(t *testing.T) {
|
||||
{"mentions-citations-midsentence", "The paper's citations are what got it retracted.", false},
|
||||
{"crisp-number", "42", false},
|
||||
{"over-cap", "Citations are logged. " + strings.Repeat("The long version has many more details worth keeping. ", 6), false}, // >300: too substantial to replace
|
||||
|
||||
// A deictic back-reference opener also qualifies — the #1611 shape,
|
||||
// which carries no citations ack at all.
|
||||
{"1611-verbatim", closer1611, true},
|
||||
{"above-pointer-plus-tldr", "That's the whole picture above. In short: the merger fell through.", true},
|
||||
{"prepositional-above-is-not-a-closer", "Anything above 100 degrees boils off, which is why the sample evaporated.", false},
|
||||
{"1611-over-cap", closer1611 + " " + strings.Repeat("Plenty more detail worth keeping here. ", 4), false}, // >300
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
@@ -147,6 +207,11 @@ func TestFinalOutput(t *testing.T) {
|
||||
// Matches BOTH the summary ack and backRefRe, within the 120-byte weak cap,
|
||||
// and long enough (>~92 bytes) that longAnswer would fail the summary bar.
|
||||
bothMatchCloser := "Citations are logged. As I mentioned above, the full detail on the money sources is in my earlier message."
|
||||
// The #1611 pair: a front-loaded analysis that dwarfs its 220-byte closer.
|
||||
analysis1611 := strings.TrimSpace(strings.Repeat("The break was the Iran strikes, then the Epstein files. ", 12)) // >3x closer1611
|
||||
// A terminal using "above" as a PREPOSITION — not a back-reference, so it
|
||||
// must survive verbatim next to a dwarfing prior turn.
|
||||
prepositionalTerminal := "Anything above 100 degrees boils off, which is exactly why the sample evaporated overnight in the unsealed tray."
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -387,6 +452,50 @@ func TestFinalOutput(t *testing.T) {
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: hugeAnswer,
|
||||
},
|
||||
{
|
||||
// mort issue #1611: a pointer-plus-compression closer with no
|
||||
// citations ack. The 2,245-char analysis was front-loaded into the
|
||||
// cite turn; the closer pointed at a "chain above" the user never
|
||||
// saw. Recover the analysis and discard the closer.
|
||||
name: "above-pointer closer discarded when the front-loaded answer dwarfs it",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("what set off the Truth Social rant?"),
|
||||
asst(analysis1611, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(closer1611),
|
||||
},
|
||||
terminal: closer1611,
|
||||
want: analysis1611,
|
||||
},
|
||||
{
|
||||
// The dwarf ratio governs the new opener too: a prior turn that is
|
||||
// longer but not clearly the fuller original (here ~275 bytes vs a
|
||||
// 220-byte closer, under 3x) must not displace a closer that
|
||||
// carries real answer content.
|
||||
name: "above-pointer closer kept when the prior turn does not dwarf it",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(closer1611),
|
||||
},
|
||||
terminal: closer1611,
|
||||
want: closer1611,
|
||||
},
|
||||
{
|
||||
// A prepositional "above" is not a back-reference: this terminal
|
||||
// stands on its own and must be returned verbatim even though a
|
||||
// much longer prior turn exists.
|
||||
name: "prepositional above is not hijacked by a longer prior turn",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("why did the sample evaporate?"),
|
||||
asst(hugeAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(prepositionalTerminal),
|
||||
},
|
||||
terminal: prepositionalTerminal,
|
||||
want: prepositionalTerminal,
|
||||
},
|
||||
{
|
||||
// A closer matching BOTH the ack shape and a back-reference
|
||||
// carries no answer content, so the back-ref test must win and the
|
||||
@@ -536,3 +645,34 @@ func TestRun_RecoversFrontLoadedAnswerWithCitations(t *testing.T) {
|
||||
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_RecoversFrontLoadedAnswerOverAboveRefCloser reproduces mort issue
|
||||
// #1611 end-to-end: the model front-loads its analysis into the cite-call turn
|
||||
// and closes with a pointer at that invisible text plus a one-line
|
||||
// compression. The delivered output must be the front-loaded analysis, with no
|
||||
// extra model call.
|
||||
func TestRun_RecoversFrontLoadedAnswerOverAboveRefCloser(t *testing.T) {
|
||||
analysis := strings.TrimSpace(strings.Repeat("The break was the Iran strikes, then the Epstein files. ", 12))
|
||||
fp := fake.New("fp")
|
||||
fp.Enqueue("test-model",
|
||||
fake.ReplyWith(llm.Response{
|
||||
Parts: []llm.Part{llm.Text(analysis)},
|
||||
ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
|
||||
FinishReason: llm.FinishToolCalls,
|
||||
Usage: llm.Usage{InputTokens: 10, OutputTokens: 5},
|
||||
}),
|
||||
fake.Reply(closer1611),
|
||||
)
|
||||
|
||||
a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
|
||||
res, err := a.Run(context.Background(), "what set off the Truth Social rant?")
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if res.Output != analysis {
|
||||
t.Errorf("Output = %q, want recovered front-loaded analysis %q", res.Output, analysis)
|
||||
}
|
||||
if n := len(fp.Calls()); n != 2 {
|
||||
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user