steveandClaude Fable 5
21b4775d16
Two behavioral fixes from the review: - modeSummary's backward scan now stops at the most recent user message. With the dwarf ratio rejecting the current turn's 1x-3x answer, the old unbounded scan could walk into WithHistory content and resurrect a stale answer to a DIFFERENT question — strictly worse than keeping the closer (opus, correctness). Other modes keep their historical unbounded scan. - A terminal matching BOTH the ack shape and a back-reference is now classified back-ref: it carries no answer content, so the looser bar is the right one (opus, error-handling). Plus the nits: summaryCloserRe assembled from named fragments, the leading marker class gains '+' (parity with citationLabelRe), verb-first form takes 'all the', dwarf ratio hoisted into one named local, and the 151-vs-153 char/byte comment inaccuracy corrected. Co-Authored-By: Claude Fable 5 <[email protected]>
539 lines
22 KiB
Go
539 lines
22 KiB
Go
package agent
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"strings"
|
||
"testing"
|
||
|
||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
|
||
)
|
||
|
||
func TestIsWeakFinal(t *testing.T) {
|
||
long := strings.Repeat("As I said, this is the full answer. ", 6) // >120, contains "as i said"
|
||
cases := []struct {
|
||
name string
|
||
in string
|
||
want bool
|
||
}{
|
||
{"empty", "", true},
|
||
{"whitespace", " \n\t ", true},
|
||
{"already-answered", "(Already answered above.)", true},
|
||
{"see-above", "see above", true},
|
||
{"as-i-said-short", "As I said, it's 60 minutes.", true},
|
||
{"crisp-number", "42", false},
|
||
{"crisp-yes", "Yes.", false},
|
||
{"crisp-status", "It's down, restarting now.", false},
|
||
{"long-with-as-i-said", long, false}, // >120 chars: not weak despite the phrase
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
if got := isWeakFinal(c.in); got != c.want {
|
||
t.Errorf("isWeakFinal(%q) = %v, want %v", c.in, got, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestIsCitationsOnly(t *testing.T) {
|
||
cases := []struct {
|
||
name string
|
||
in string
|
||
want bool
|
||
}{
|
||
{"sources-md-links", "Sources: [pcprice.watch](https://pcprice.watch/x), [ebay](https://ebay.com/1).", true},
|
||
{"lowercase-bare-url", "sources: see https://example.com/a", true},
|
||
{"bold-label-colon-inside", "**Sources:** [a](https://a), [b](https://b)", true},
|
||
{"bold-label-colon-outside", "**Sources**: [a](https://a), [b](https://b)", true}, // colon after the closing **
|
||
{"references-dash", "References — [a](https://a)", true},
|
||
{"citations-label", "Citations: https://x/y", true},
|
||
{"leading-list-marker", "- Sources: [a](https://a)", true},
|
||
{"atx-heading", "## Sources: [a](https://a), [b](https://b)", true}, // ATX heading marker + its trailing space
|
||
{"further-reading", "Further reading: https://example.com/deep-dive", true},
|
||
{"annotated-multi-source", "Sources: [pcprice.watch](https://a) (tracker), [eBay](https://b) (sold), [bestvaluegpu](https://c) (retail), [resaleprices](https://d) (asking).", true}, // the reported issue-1418 shape
|
||
{"backref-plus-links-is-citations", "References: as noted above, [pcprice.watch](https://pcprice.watch/x).", true}, // a back-ref phrase inside a real sources list is still citations
|
||
|
||
{"empty", "", false},
|
||
{"label-but-no-link", "Source: internal analysis, no URL here", false},
|
||
{"prose-then-sources", "It sells for ~$2,700. Sources: [a](https://a)", false}, // answer first → not a pure addendum
|
||
{"source-led-prose-answer", "Source: According to https://cdc.gov the flu vaccine is 40-60% effective, and the CDC recommends annual vaccination for everyone over six months old.", false}, // a prose answer that merely opens with a "Source:" label
|
||
{"mentions-sources-midsentence", "The sources of the leak were never confirmed.", false},
|
||
{"link-without-label", "Here is the link you asked for: [a](https://a)", false},
|
||
{"bare-domains-out-of-scope", "Sources: pcprice.watch (used ~$200), ebay.com (sold listings)", false}, // bare domains: no scheme or markdown link to key on
|
||
{"crisp-number", "42", false},
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
if got := isCitationsOnly(c.in); got != c.want {
|
||
t.Errorf("isCitationsOnly(%q) = %v, want %v", c.in, got, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// b3cb9ee9Closer is the verbatim terminal turn from mort run b3cb9ee9: a
|
||
// 2,089-char answer was front-loaded into the cite-call turn and this 153-byte
|
||
// compression (151 runes — the em dash is 3 bytes, and byte length is what the
|
||
// thresholds compare) was all that got delivered.
|
||
const b3cb9ee9Closer = "Citations are logged. Short version: the bulk of that ~$64M was AIPAC and dark-money super PACs, not the party committees — and it still wasn't enough."
|
||
|
||
func TestIsSummaryCloser(t *testing.T) {
|
||
cases := []struct {
|
||
name string
|
||
in string
|
||
want bool
|
||
}{
|
||
{"b3cb9ee9-verbatim", b3cb9ee9Closer, true},
|
||
{"ack-only", "Citations are logged.", true},
|
||
{"ack-no-copula", "Citations logged.", true},
|
||
{"claims-cited", "All claims cited.", true},
|
||
{"verb-first", "Logged the citations.", true},
|
||
{"done-prefix", "Done — citations logged.", true},
|
||
{"ack-then-tldr", "Sources have been recorded! TL;DR: the GPU was the bottleneck.", true},
|
||
{"references-noted", "References noted. In short: yes, it ships Tuesday.", true},
|
||
{"plus-list-marker", "+ Citations are logged.", true},
|
||
{"logged-all-the", "Logged all the citations.", true},
|
||
|
||
{"empty", "", false},
|
||
{"ack-continues-midsentence", "The citations are recorded in the court transcript, which shows the filing dates.", false},
|
||
{"ack-verb-then-clause", "Citations are logged in Zotero whenever you click the save button.", false},
|
||
{"compression-without-ack", "Short version: yes.", false}, // deliberately out of scope
|
||
{"mentions-citations-midsentence", "The paper's citations are what got it retracted.", false},
|
||
{"crisp-number", "42", false},
|
||
{"over-cap", "Citations are logged. " + strings.Repeat("The long version has many more details worth keeping. ", 6), false}, // >300: too substantial to replace
|
||
}
|
||
for _, c := range cases {
|
||
t.Run(c.name, func(t *testing.T) {
|
||
if got := isSummaryCloser(c.in); got != c.want {
|
||
t.Errorf("isSummaryCloser(%q) = %v, want %v", c.in, got, c.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func asst(text string, tools ...llm.ToolCall) llm.Message {
|
||
m := llm.Message{Role: llm.RoleAssistant}
|
||
if text != "" {
|
||
m.Parts = []llm.Part{llm.Text(text)}
|
||
}
|
||
m.ToolCalls = tools
|
||
return m
|
||
}
|
||
|
||
func TestFinalOutput(t *testing.T) {
|
||
cite := []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}}
|
||
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6)) // >200
|
||
hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12)) // >3x the b3cb9ee9 closer
|
||
// A sources/citations-only terminal — the glm-5.2 "cite" shape behind mort
|
||
// issue #1418: the prose answer was front-loaded into the tool-call turn and
|
||
// the terminal turn carried only the citations.
|
||
sources := "Sources: [pcprice.watch](https://pcprice.watch/x), [ebay](https://www.ebay.com/itm/1)."
|
||
answerWithSources := longAnswer + "\n\n" + sources
|
||
// A concise (>80, <200 byte) front-loaded answer + a long citations terminal:
|
||
// the ratio arm can't be met against the long terminal, so citations mode
|
||
// must fall back to the floor.
|
||
conciseAnswer := "It sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle."
|
||
longSources := "Sources: [pcprice.watch](https://pcprice.watch/gpu/rtx5090) (tracker), [ebay](https://www.ebay.com/sch/rtx5090) (sold), [newegg](https://newegg.com/rtx5090) (retail), [pcpartpicker](https://pcpartpicker.com/rtx5090) (history)."
|
||
// A substantive answer that merely OPENS with "Source:" (not a bare list).
|
||
sourceLedAnswer := "Source: https://nvd.nist.gov/vuln/detail/CVE-2024-1234 — this is the authoritative NVD entry for the vulnerability, rated CVSS 9.8 critical."
|
||
// A borderline-band (80–200 byte) turn that opens like a planning preamble:
|
||
// it clears the floor, but the preamble filter still vetoes it (the filter
|
||
// applies only in the borderline band; a >=200-byte turn is accepted as-is).
|
||
preambleTurn := "Let me look that up across a few different sites and then compile the full comparison for you here."
|
||
// A >=200-byte real answer that merely OPENS with a conversational word
|
||
// ("Sure,"). The preamble filter must NOT veto it (gadfly regression guard).
|
||
longConversationalAnswer := "Sure, here's the rundown: it currently sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle, with the sealed Founders Edition commanding the top of that range while used AIB cards go a bit lower."
|
||
// Matches BOTH the summary ack and backRefRe, within the 120-byte weak cap,
|
||
// and long enough (>~92 bytes) that longAnswer would fail the summary bar.
|
||
bothMatchCloser := "Citations are logged. As I mentioned above, the full detail on the money sources is in my earlier message."
|
||
|
||
tests := []struct {
|
||
name string
|
||
msgs []llm.Message
|
||
terminal string
|
||
want string
|
||
}{
|
||
{
|
||
name: "front-loaded answer recovered over back-ref closer",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(longAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst("(Already answered above.)"),
|
||
},
|
||
terminal: "(Already answered above.)",
|
||
want: longAnswer,
|
||
},
|
||
{
|
||
name: "empty terminal recovers prior substantive answer",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(longAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(""),
|
||
},
|
||
terminal: "",
|
||
want: longAnswer,
|
||
},
|
||
{
|
||
name: "healthy terminal answer is unchanged",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst("Let me check.", cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(longAnswer),
|
||
},
|
||
terminal: longAnswer,
|
||
want: longAnswer,
|
||
},
|
||
{
|
||
name: "short crisp answer not overridden by a short preamble prior",
|
||
msgs: []llm.Message{
|
||
llm.UserText("is it up?"),
|
||
asst("Let me check the server status.", cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst("It's down, restarting now."),
|
||
},
|
||
terminal: "It's down, restarting now.", // not weak → returned as-is
|
||
want: "It's down, restarting now.",
|
||
},
|
||
{
|
||
name: "weak terminal but only a preamble prior: no recovery",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst("Let me look that up for you.", cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst("(see above)"),
|
||
},
|
||
terminal: "(see above)",
|
||
want: "(see above)", // preamble excluded; falls back to terminal
|
||
},
|
||
{
|
||
name: "citations-only terminal recovers front-loaded answer and keeps sources",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(longAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(sources),
|
||
},
|
||
terminal: sources,
|
||
want: answerWithSources, // answer recovered, citations appended
|
||
},
|
||
{
|
||
name: "citations-only terminal but only a preamble prior: keeps the sources",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst("Let me gather the sources.", cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(sources),
|
||
},
|
||
terminal: sources,
|
||
want: sources, // nothing substantive to recover → keep the addendum
|
||
},
|
||
{
|
||
name: "citations already in the recovered answer are not duplicated",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(answerWithSources, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(sources),
|
||
},
|
||
terminal: sources,
|
||
want: answerWithSources, // recovered turn already carries the sources
|
||
},
|
||
{
|
||
// #1418 persisted for CONCISE answers: a <200-char front-loaded
|
||
// answer must still be recovered against a long citations terminal
|
||
// (the ratio arm is skipped in citations mode).
|
||
name: "concise front-loaded answer recovered against a long citations terminal",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(conciseAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(longSources),
|
||
},
|
||
terminal: longSources,
|
||
want: conciseAnswer + "\n\n" + longSources,
|
||
},
|
||
{
|
||
// A substantive answer that merely OPENS with "Source:" and cites a
|
||
// URL mid-sentence is NOT a citations addendum — return it verbatim,
|
||
// never prepend the prior planning turn.
|
||
name: "source-led substantive answer is not hijacked by a prior turn",
|
||
msgs: []llm.Message{
|
||
llm.UserText("what's the authoritative URL?"),
|
||
asst("I'll look up the CVE in the NVD database, cross-reference the vendor advisory, and confirm the canonical URL before I answer.", cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(sourceLedAnswer),
|
||
},
|
||
terminal: sourceLedAnswer,
|
||
want: sourceLedAnswer,
|
||
},
|
||
{
|
||
// A borderline-length turn that opens like a preamble is vetoed
|
||
// during recovery; the older real answer is recovered instead. (A
|
||
// >=200-byte turn would be accepted verbatim — see the next case.)
|
||
name: "borderline preamble is skipped; older real answer recovered",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(conciseAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(preambleTurn, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c2", Name: "cite", Content: "ok"}),
|
||
asst(sources),
|
||
},
|
||
terminal: sources,
|
||
want: conciseAnswer + "\n\n" + sources,
|
||
},
|
||
{
|
||
// Guards the gadfly regression: a LONG (>=200-byte) front-loaded
|
||
// answer that merely opens with a conversational word ("Sure, …")
|
||
// must still be recovered — the preamble filter must not veto it.
|
||
name: "long answer opening with a conversational word is still recovered",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(longConversationalAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(sources),
|
||
},
|
||
terminal: sources,
|
||
want: longConversationalAnswer + "\n\n" + sources,
|
||
},
|
||
{
|
||
// The b3cb9ee9 shape: full answer front-loaded into the cite turn,
|
||
// then a summary closer. The closer is discarded — its content is a
|
||
// strict compression of the recovered answer.
|
||
name: "summary closer discarded when the front-loaded answer dwarfs it",
|
||
msgs: []llm.Message{
|
||
llm.UserText("where did the $64M come from?"),
|
||
asst(hugeAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(b3cb9ee9Closer),
|
||
},
|
||
terminal: b3cb9ee9Closer,
|
||
want: hugeAnswer,
|
||
},
|
||
{
|
||
// The dwarf ratio is mandatory for a summary closer at EVERY length:
|
||
// a prior turn that is longer but not clearly the fuller original
|
||
// (here ~275 chars vs a 151-char closer, under the 3x bar) must not
|
||
// displace a closer that carries real answer content.
|
||
name: "summary closer kept when the prior turn does not dwarf it",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(longAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(b3cb9ee9Closer),
|
||
},
|
||
terminal: b3cb9ee9Closer,
|
||
want: b3cb9ee9Closer,
|
||
},
|
||
{
|
||
// An ack-only closer ("Citations are logged.") is tiny, so even a
|
||
// modest front-loaded answer clears the ratio and replaces it.
|
||
name: "ack-only summary closer recovered over a modest answer",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(longAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst("Citations are logged."),
|
||
},
|
||
terminal: "Citations are logged.",
|
||
want: longAnswer,
|
||
},
|
||
{
|
||
name: "summary closer with only a preamble prior keeps the closer",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst("Let me gather the numbers.", cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(b3cb9ee9Closer),
|
||
},
|
||
terminal: b3cb9ee9Closer,
|
||
want: b3cb9ee9Closer,
|
||
},
|
||
{
|
||
// Gadfly (opus, correctness): the modeSummary scan must stop at the
|
||
// most recent user message. Here the current turn's answer sits in
|
||
// the 1x-3x band (rejected by the ratio) while a dwarfing answer to
|
||
// a DIFFERENT question sits in history — resurrecting it would be
|
||
// strictly worse than keeping the closer.
|
||
name: "summary closer never resurrects a stale answer across the user boundary",
|
||
msgs: []llm.Message{
|
||
llm.UserText("earlier, unrelated question?"),
|
||
asst(hugeAnswer),
|
||
llm.UserText("q?"),
|
||
asst(conciseAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(b3cb9ee9Closer),
|
||
},
|
||
terminal: b3cb9ee9Closer,
|
||
want: b3cb9ee9Closer,
|
||
},
|
||
{
|
||
// The boundary must not break the legitimate multi-turn case: the
|
||
// dwarfing front-loaded answer in THIS turn's window is recovered
|
||
// even with history behind it.
|
||
name: "summary closer recovery still works with history present",
|
||
msgs: []llm.Message{
|
||
llm.UserText("earlier, unrelated question?"),
|
||
asst(longAnswer),
|
||
llm.UserText("q?"),
|
||
asst(hugeAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(b3cb9ee9Closer),
|
||
},
|
||
terminal: b3cb9ee9Closer,
|
||
want: hugeAnswer,
|
||
},
|
||
{
|
||
// Gadfly (opus, error-handling): a closer matching BOTH the ack
|
||
// shape and a back-reference carries no answer content, so the
|
||
// back-ref test must win and the ordinary recovery bar apply —
|
||
// under the summary bar this ~106-byte terminal would demand a
|
||
// ~318-byte prior and wrongly keep the closer over longAnswer.
|
||
name: "back-reference wins over the summary ack when both match",
|
||
msgs: []llm.Message{
|
||
llm.UserText("q?"),
|
||
asst(longAnswer, cite...),
|
||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||
asst(bothMatchCloser),
|
||
},
|
||
terminal: bothMatchCloser,
|
||
want: longAnswer,
|
||
},
|
||
}
|
||
for _, tc := range tests {
|
||
t.Run(tc.name, func(t *testing.T) {
|
||
if got := finalOutput(tc.msgs, tc.terminal); got != tc.want {
|
||
t.Errorf("finalOutput = %q, want %q", got, tc.want)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
func citeToolbox(t *testing.T) *llm.Toolbox {
|
||
t.Helper()
|
||
return llm.NewToolbox("sources", llm.Tool{
|
||
Name: "cite",
|
||
Description: "Record a citation.",
|
||
Parameters: json.RawMessage(`{"type":"object","properties":{}}`),
|
||
Handler: func(_ context.Context, _ json.RawMessage) (any, error) {
|
||
return map[string]bool{"ok": true}, nil
|
||
},
|
||
})
|
||
}
|
||
|
||
// TestRun_RecoversFrontLoadedAnswer reproduces the glm-5.2 shape end-to-end: a
|
||
// turn carrying the full answer text AND a tool call, then a degenerate
|
||
// terminal turn. The recovered answer must be delivered with no extra model
|
||
// call (zero-cost recovery from the transcript).
|
||
func TestRun_RecoversFrontLoadedAnswer(t *testing.T) {
|
||
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6))
|
||
fp := fake.New("fp")
|
||
fp.Enqueue("test-model",
|
||
fake.ReplyWith(llm.Response{
|
||
Parts: []llm.Part{llm.Text(longAnswer)},
|
||
ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
|
||
FinishReason: llm.FinishToolCalls,
|
||
Usage: llm.Usage{InputTokens: 10, OutputTokens: 5},
|
||
}),
|
||
fake.Reply("(Already answered above.)"),
|
||
)
|
||
|
||
a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
|
||
res, err := a.Run(context.Background(), "is there a meet time limit?")
|
||
if err != nil {
|
||
t.Fatalf("Run: %v", err)
|
||
}
|
||
if res.Output != longAnswer {
|
||
t.Errorf("Output = %q, want recovered front-loaded answer", res.Output)
|
||
}
|
||
if n := len(fp.Calls()); n != 2 {
|
||
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||
}
|
||
}
|
||
|
||
// TestRun_HealthyTerminalUnchanged guards against regressing the normal case:
|
||
// a deferred answer in the terminal turn is delivered verbatim.
|
||
func TestRun_HealthyTerminalUnchanged(t *testing.T) {
|
||
fp := fake.New("fp")
|
||
fp.Enqueue("test-model",
|
||
toolCallReply("c1", "cite", `{}`),
|
||
fake.Reply("The limit is 60 minutes for free group calls."),
|
||
)
|
||
a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
|
||
res, err := a.Run(context.Background(), "q?")
|
||
if err != nil {
|
||
t.Fatalf("Run: %v", err)
|
||
}
|
||
if res.Output != "The limit is 60 minutes for free group calls." {
|
||
t.Errorf("Output = %q, want terminal answer unchanged", res.Output)
|
||
}
|
||
}
|
||
|
||
// TestRun_RecoversFrontLoadedAnswerOverSummaryCloser reproduces mort run
|
||
// b3cb9ee9 end-to-end: the model front-loads its full answer into the
|
||
// cite-call turn, the cite results come back, and the terminal turn is only a
|
||
// bookkeeping ack plus a one-line compression. The delivered output must be
|
||
// the front-loaded answer, with no extra model call.
|
||
func TestRun_RecoversFrontLoadedAnswerOverSummaryCloser(t *testing.T) {
|
||
hugeAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 12))
|
||
fp := fake.New("fp")
|
||
fp.Enqueue("test-model",
|
||
fake.ReplyWith(llm.Response{
|
||
Parts: []llm.Part{llm.Text(hugeAnswer)},
|
||
ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
|
||
FinishReason: llm.FinishToolCalls,
|
||
Usage: llm.Usage{InputTokens: 10, OutputTokens: 5},
|
||
}),
|
||
fake.Reply(b3cb9ee9Closer),
|
||
)
|
||
|
||
a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
|
||
res, err := a.Run(context.Background(), "where did the $64M come from?")
|
||
if err != nil {
|
||
t.Fatalf("Run: %v", err)
|
||
}
|
||
if res.Output != hugeAnswer {
|
||
t.Errorf("Output = %q, want recovered front-loaded answer", res.Output)
|
||
}
|
||
if n := len(fp.Calls()); n != 2 {
|
||
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||
}
|
||
}
|
||
|
||
// TestRun_RecoversFrontLoadedAnswerWithCitations reproduces mort issue #1418
|
||
// end-to-end: the model front-loads the prose answer into the tool-call turn
|
||
// and closes with a sources-only terminal turn. The delivered output must be
|
||
// the recovered answer with the citations appended (not the bare sources list),
|
||
// with no extra model call.
|
||
func TestRun_RecoversFrontLoadedAnswerWithCitations(t *testing.T) {
|
||
longAnswer := strings.TrimSpace(strings.Repeat("Free group calls are capped at sixty minutes. ", 6))
|
||
sources := "Sources: [docs](https://example.com/docs), [pricing](https://example.com/pricing)."
|
||
fp := fake.New("fp")
|
||
fp.Enqueue("test-model",
|
||
fake.ReplyWith(llm.Response{
|
||
Parts: []llm.Part{llm.Text(longAnswer)},
|
||
ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
|
||
FinishReason: llm.FinishToolCalls,
|
||
Usage: llm.Usage{InputTokens: 10, OutputTokens: 5},
|
||
}),
|
||
fake.Reply(sources),
|
||
)
|
||
|
||
a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
|
||
res, err := a.Run(context.Background(), "is there a meet time limit?")
|
||
if err != nil {
|
||
t.Fatalf("Run: %v", err)
|
||
}
|
||
want := longAnswer + "\n\n" + sources
|
||
if res.Output != want {
|
||
t.Errorf("Output = %q, want recovered answer + citations %q", res.Output, want)
|
||
}
|
||
if n := len(fp.Calls()); n != 2 {
|
||
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||
}
|
||
}
|