test(agent): guard the no-preamble-veto-for-long-answers case; reword comments (gadfly)
Add "long answer opening with a conversational word is still recovered" (regression guard); retarget the skipped-preamble case to the borderline band; drop the external "#N" annotations in favor of self-explanatory comments. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+32
-13
@@ -45,22 +45,22 @@ func TestIsCitationsOnly(t *testing.T) {
|
||||
{"sources-md-links", "Sources: [pcprice.watch](https://pcprice.watch/x), [ebay](https://ebay.com/1).", true},
|
||||
{"lowercase-bare-url", "sources: see https://example.com/a", true},
|
||||
{"bold-label-colon-inside", "**Sources:** [a](https://a), [b](https://b)", true},
|
||||
{"bold-label-colon-outside", "**Sources**: [a](https://a), [b](https://b)", true}, // #4: colon after the bold
|
||||
{"bold-label-colon-outside", "**Sources**: [a](https://a), [b](https://b)", true}, // colon after the closing **
|
||||
{"references-dash", "References — [a](https://a)", true},
|
||||
{"citations-label", "Citations: https://x/y", true},
|
||||
{"leading-list-marker", "- Sources: [a](https://a)", true},
|
||||
{"atx-heading", "## Sources: [a](https://a), [b](https://b)", true}, // #8: heading + its space
|
||||
{"atx-heading", "## Sources: [a](https://a), [b](https://b)", true}, // ATX heading marker + its trailing space
|
||||
{"further-reading", "Further reading: https://example.com/deep-dive", true},
|
||||
{"annotated-multi-source", "Sources: [pcprice.watch](https://a) (tracker), [eBay](https://b) (sold), [bestvaluegpu](https://c) (retail), [resaleprices](https://d) (asking).", true}, // the real #1418 shape
|
||||
{"backref-plus-links-is-citations", "References: as noted above, [pcprice.watch](https://pcprice.watch/x).", true}, // #13: still a citations addendum
|
||||
{"annotated-multi-source", "Sources: [pcprice.watch](https://a) (tracker), [eBay](https://b) (sold), [bestvaluegpu](https://c) (retail), [resaleprices](https://d) (asking).", true}, // the reported issue-1418 shape
|
||||
{"backref-plus-links-is-citations", "References: as noted above, [pcprice.watch](https://pcprice.watch/x).", true}, // a back-ref phrase inside a real sources list is still citations
|
||||
|
||||
{"empty", "", false},
|
||||
{"label-but-no-link", "Source: internal analysis, no URL here", false},
|
||||
{"prose-then-sources", "It sells for ~$2,700. Sources: [a](https://a)", false}, // answer first → not a pure addendum
|
||||
{"source-led-prose-answer", "Source: According to https://cdc.gov the flu vaccine is 40-60% effective, and the CDC recommends annual vaccination for everyone over six months old.", false}, // #5: prose that merely opens with a label
|
||||
{"source-led-prose-answer", "Source: According to https://cdc.gov the flu vaccine is 40-60% effective, and the CDC recommends annual vaccination for everyone over six months old.", false}, // a prose answer that merely opens with a "Source:" label
|
||||
{"mentions-sources-midsentence", "The sources of the leak were never confirmed.", false},
|
||||
{"link-without-label", "Here is the link you asked for: [a](https://a)", false},
|
||||
{"bare-domains-out-of-scope", "Sources: pcprice.watch (used ~$200), ebay.com (sold listings)", false}, // #7: no scheme/link signal
|
||||
{"bare-domains-out-of-scope", "Sources: pcprice.watch (used ~$200), ebay.com (sold listings)", false}, // bare domains: no scheme or markdown link to key on
|
||||
{"crisp-number", "42", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
@@ -96,8 +96,13 @@ func TestFinalOutput(t *testing.T) {
|
||||
longSources := "Sources: [pcprice.watch](https://pcprice.watch/gpu/rtx5090) (tracker), [ebay](https://www.ebay.com/sch/rtx5090) (sold), [newegg](https://newegg.com/rtx5090) (retail), [pcpartpicker](https://pcpartpicker.com/rtx5090) (history)."
|
||||
// A substantive answer that merely OPENS with "Source:" (not a bare list).
|
||||
sourceLedAnswer := "Source: https://nvd.nist.gov/vuln/detail/CVE-2024-1234 — this is the authoritative NVD entry for the vulnerability, rated CVSS 9.8 critical."
|
||||
// A >=200-byte planning preamble (must be skipped during recovery).
|
||||
longPreamble := "I'll look up the current eBay sold listings, then cross-reference the pcprice.watch tracker and a couple of retail sources, compare the medians across all of them, and put together a clear price range for you before I give the final number."
|
||||
// A borderline-band (80–200 byte) turn that opens like a planning preamble:
|
||||
// it clears the floor, but the preamble filter still vetoes it (the filter
|
||||
// applies only in the borderline band; a >=200-byte turn is accepted as-is).
|
||||
preambleTurn := "Let me look that up across a few different sites and then compile the full comparison for you here."
|
||||
// A >=200-byte real answer that merely OPENS with a conversational word
|
||||
// ("Sure,"). The preamble filter must NOT veto it (gadfly regression guard).
|
||||
longConversationalAnswer := "Sure, here's the rundown: it currently sells for about $2,700 used on eBay, typically $2,400 to $2,900 depending on condition and bundle, with the sealed Founders Edition commanding the top of that range while used AIB cards go a bit lower."
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -222,21 +227,35 @@ func TestFinalOutput(t *testing.T) {
|
||||
want: sourceLedAnswer,
|
||||
},
|
||||
{
|
||||
// A long (>=200) planning preamble must be skipped during recovery
|
||||
// (preamble check applies before the length shortcut); the older
|
||||
// real answer is recovered instead.
|
||||
name: "long preamble is skipped; older real answer recovered",
|
||||
// A borderline-length turn that opens like a preamble is vetoed
|
||||
// during recovery; the older real answer is recovered instead. (A
|
||||
// >=200-byte turn would be accepted verbatim — see the next case.)
|
||||
name: "borderline preamble is skipped; older real answer recovered",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(conciseAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(longPreamble, cite...),
|
||||
asst(preambleTurn, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c2", Name: "cite", Content: "ok"}),
|
||||
asst(sources),
|
||||
},
|
||||
terminal: sources,
|
||||
want: conciseAnswer + "\n\n" + sources,
|
||||
},
|
||||
{
|
||||
// Guards the gadfly regression: a LONG (>=200-byte) front-loaded
|
||||
// answer that merely opens with a conversational word ("Sure, …")
|
||||
// must still be recovered — the preamble filter must not veto it.
|
||||
name: "long answer opening with a conversational word is still recovered",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longConversationalAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(sources),
|
||||
},
|
||||
terminal: sources,
|
||||
want: longConversationalAnswer + "\n\n" + sources,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user