test(agent): gadfly round 1b — pin the exact bound and the bar asymmetry
Sonnet's three findings, all on the tests rather than the rule: - The "at the bound" case sat at index 115 (and 101 after the first round), never at 120 itself, so <= vs < was never exercised. Both sides of the boundary are now pinned — index 120 accepted, 121 rejected — and an off-by-one mutation is killed by the new case. - A SHORT (<=120-byte) deictic closer takes the modeBackRef bar, not the summary closer's mandatory dwarf ratio, even though it can carry a scrap of answer content. That asymmetry is deliberate and pre-existing (a "see above" closer has always had it): inside 120 bytes there is no room for both a pointer and a real answer. Pinned with a case sized so the two bars actually DISAGREE — 3x98 = 294 > the 275-byte prior turn, so the summary bar would reject what the back-ref bar accepts. The first draft of this fixture was 83 bytes and passed under either bar, proving nothing. - The isBackRef doc finding was sonnet's fourth independent report of it; already fixed in the previous commit. Break-check now runs eight mutations, including the off-by-one and a swap of the classifier ordering; each is killed by a named test and the control survives.
This commit is contained in:
+31
-1
@@ -96,7 +96,12 @@ func TestPointsAbove(t *testing.T) {
|
||||
// the constant it is meant to pin moves with it, and a break-check
|
||||
// that widened the bound to 100000 sailed straight through.
|
||||
{"late-reference-not-a-pointer", strings.Repeat("x", 200) + " as shown above.", false},
|
||||
{"reference-at-the-bound", strings.Repeat("x", 100) + " above.", true},
|
||||
// The exact boundary, both sides: "above" starts at index 120 (allowed,
|
||||
// the bound is <=) and at 121 (rejected). 119/120 x's plus the space
|
||||
// puts the 'a' on 120/121 — the separator is needed because \b will not
|
||||
// hold between "x" and "above".
|
||||
{"reference-exactly-at-the-bound", strings.Repeat("x", 119) + " above.", true},
|
||||
{"reference-one-past-the-bound", strings.Repeat("x", 120) + " above.", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
@@ -239,6 +244,12 @@ func TestFinalOutput(t *testing.T) {
|
||||
bothMatchCloser := "Citations are logged. As I mentioned above, the full detail on the money sources is in my earlier message."
|
||||
// The #1611 pair: a front-loaded analysis that dwarfs its 220-byte closer.
|
||||
analysis := analysis1611()
|
||||
// A 98-byte deictic closer: inside the weak-final cap (120), and sized so
|
||||
// the two bars actually DISAGREE about it — 3x98 = 294 > longAnswer's 275,
|
||||
// so the summary closer's dwarf ratio would reject longAnswer while the
|
||||
// modeBackRef bar (>=200 bytes, no ratio) accepts it. A shorter closer
|
||||
// would pass under either bar and prove nothing.
|
||||
shortAbovePointer := "Done. The whole chain is above, so there is no point repeating all of that detail down here again."
|
||||
// A terminal using "above" as a PREPOSITION — not a back-reference, so it
|
||||
// must survive verbatim next to a dwarfing prior turn.
|
||||
prepositionalTerminal := "Anything above 100 degrees boils off, which is exactly why the sample evaporated overnight in the unsealed tray."
|
||||
@@ -526,6 +537,25 @@ func TestFinalOutput(t *testing.T) {
|
||||
terminal: prepositionalTerminal,
|
||||
want: prepositionalTerminal,
|
||||
},
|
||||
{
|
||||
// A SHORT deictic closer takes the modeBackRef bar, not the summary
|
||||
// closer's mandatory dwarf ratio — even though it carries a scrap
|
||||
// of answer content ("Done."). Deliberate, and the same contract a
|
||||
// short "see above" closer has always had: within the 120-byte cap
|
||||
// there is no room for both a pointer and a real answer, so a
|
||||
// >=200-byte prior turn wins without having to be 3x. Here the
|
||||
// ratio would demand ~330 bytes and wrongly keep the pointer.
|
||||
// (gadfly/sonnet flagged the asymmetry; this pins it.)
|
||||
name: "short above-pointer closer uses the back-ref bar, not the dwarf ratio",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(shortAbovePointer),
|
||||
},
|
||||
terminal: shortAbovePointer,
|
||||
want: longAnswer,
|
||||
},
|
||||
{
|
||||
// A closer matching BOTH the ack shape and a back-reference
|
||||
// carries no answer content, so the back-ref test must win and the
|
||||
|
||||
Reference in New Issue
Block a user