Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1756910ef0 | ||
|
|
3f1c016e74 | ||
|
|
7dccb233ad | ||
|
|
9534e442fb | ||
|
|
f97c2b78c2 | ||
|
|
bcba9667bd |
@@ -42,17 +42,7 @@ jobs:
|
||||
# and cache the reusable-workflow ref, so a moved v1 tag keeps resolving to the
|
||||
# stale cached copy. A unique sha forces a cache miss → fresh fetch. Bump this
|
||||
# sha to adopt central swarm changes.
|
||||
#
|
||||
# NB: the reviewer IMAGE tag is no longer part of this pin. From @8adeeea on,
|
||||
# the reusable workflow resolves it per run as
|
||||
# `inputs.reviewer_tag || vars.GADFLY_REVIEWER_TAG || 'sha-b850e35'`, so
|
||||
# retagging the reviewer is a user-scope variable edit with no commit here.
|
||||
# The previous pin (@c9dab69d) hardcoded `docker://…gadfly:sha-b37cd09`, a tag
|
||||
# that was never pushed to the registry — every review in this repo died in
|
||||
# ~1s on "failed to resolve reference … not found", which reads exactly like a
|
||||
# review that found nothing. Before bumping this pin again, check the tag its
|
||||
# fallback names is really in the registry.
|
||||
uses: steve/gadfly/.gitea/workflows/review-reusable.yml@8adeeeabe0738a797a1bdfc42c5176ea8ee627e4
|
||||
uses: steve/gadfly/.gitea/workflows/review-reusable.yml@c9dab69d143cb614c1840a5b06d6ffc358f4752d
|
||||
# Least privilege: forward only the review secrets (not `secrets: inherit`,
|
||||
# which would expose every repo secret). GITEA_TOKEN is the automatic token.
|
||||
secrets:
|
||||
|
||||
+169
-20
@@ -25,14 +25,21 @@ import (
|
||||
// glm-5.2 "cite" pattern behind mort issue #1418). The citations are real,
|
||||
// useful content — unlike a back-reference — so recover the prior answer and
|
||||
// KEEP the citations, appended below it.
|
||||
// - a bookkeeping closer ("Citations are logged. Short version: …"): the
|
||||
// model acknowledged the citation round and compressed the answer it had
|
||||
// already written into a one-liner (mort run b3cb9ee9 — a 2,089-char answer
|
||||
// shrank to a 153-byte closer at delivery). The compression is strictly
|
||||
// poorer than the front-loaded answer, so recover the prior turn and
|
||||
// DISCARD the closer — but only when the prior turn clearly dwarfs it,
|
||||
// because unlike a back-reference this closer DOES carry answer content
|
||||
// (see modeSummary).
|
||||
// - a deferring closer ("Citations are logged. Short version: …", "Done —
|
||||
// that's the full chain above. Short version: …"): the model points at the
|
||||
// answer it had already written and compresses it into a one-liner (mort
|
||||
// run b3cb9ee9 — a 2,089-char answer shrank to a 153-byte closer; mort
|
||||
// issue #1611 — a 2,245-char analysis shrank to 220 bytes). The
|
||||
// compression is strictly poorer than the front-loaded answer, so recover
|
||||
// the prior turn and DISCARD the closer — but only when the prior turn
|
||||
// clearly dwarfs it, because unlike a bare back-reference this closer DOES
|
||||
// carry answer content (see modeSummary).
|
||||
//
|
||||
// The last two shapes share one signal — the terminal DEFERS: it tells us the
|
||||
// answer is somewhere the user cannot see (pointsAbove, or the bookkeeping
|
||||
// ack). They differ only in whether the terminal also carries content of its
|
||||
// own, so a new deferral phrasing added to pointsAbove is covered in both the
|
||||
// bare and the "+ compression" variant at once.
|
||||
//
|
||||
// A citations addendum is tested first and wins over the other two (a short
|
||||
// terminal can match more than one shape), so its links are never discarded.
|
||||
@@ -110,9 +117,109 @@ const (
|
||||
|
||||
// backRefRe matches a terminal turn that merely points back to an earlier
|
||||
// message instead of stating the answer ("(Already answered above.)",
|
||||
// "see above", "as I said", ...).
|
||||
// "see above", "as I said", ...). It is a list of fixed phrasings; aboveRefRe
|
||||
// covers the open-ended half of the same family.
|
||||
var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(said|mentioned|stated|noted)|answered (that )?above|per my (previous|earlier))`)
|
||||
|
||||
// aboveRefRe matches a DEICTIC "above" — one pointing at earlier text rather
|
||||
// than serving as a preposition. What follows the word separates the two uses:
|
||||
// the deictic use ends its clause ("that's the full chain above.", "as shown
|
||||
// above,", a line that simply ends in "above"), while the preposition always
|
||||
// continues into a noun phrase ("above 100°C", "above the fold", "above all,
|
||||
// the ..."). Only the clause-final form matches, so the open-ended half of the
|
||||
// back-reference family is covered without enumerating every phrasing a model
|
||||
// might invent — backRefRe's fixed list kept missing new ones (mort issue
|
||||
// #1611: "Done — that's the full chain above.").
|
||||
//
|
||||
// The terminator set deliberately excludes the ASCII hyphen. `\b` holds
|
||||
// between "above" and "-", so a literal '-' in the class made every
|
||||
// hyphenated compound — "above-average", "above-board", "above-ground" —
|
||||
// read as a clause-final deictic and put a legitimate short answer at risk of
|
||||
// being discarded. The em dash stays: a model writes "…above — see the
|
||||
// links", never "above-" as a separator. CR is listed alongside LF so a
|
||||
// CRLF transcript does not quietly lose every line-final "above".
|
||||
var aboveRefRe = regexp.MustCompile(`(?i)\babove\b[ \t]*([.,;:!?)\]"'’”—]|\r|\n|$)`)
|
||||
|
||||
// pointsAbove reports whether a deictic "above" appears in the terminal's
|
||||
// OPENING. The offset bound is what makes a bare "above" safe to key on: with
|
||||
// almost no text before it in THIS message, the reference cannot be pointing
|
||||
// at the message's own content, so it must point at a turn the user never saw
|
||||
// (the harness delivers only the final turn).
|
||||
//
|
||||
// This is the DEFERRAL SIGNAL SHARED by both recovery shapes — isWeakFinal and
|
||||
// isSummaryCloser both call it, so a widening here reaches the bare closer and
|
||||
// the "+ compression" closer at once. Its sibling backRefRe is deliberately
|
||||
// NOT shared: those fixed phrases are matched anywhere in the text, which is
|
||||
// only safe under isWeakFinal's 120-byte cap. Folding the two into one
|
||||
// predicate for tidiness would hand isSummaryCloser an unanchored match across
|
||||
// 300 bytes — widening the gate, not deduplicating it.
|
||||
func pointsAbove(t string) bool {
|
||||
loc := aboveRefRe.FindStringIndex(t)
|
||||
return loc != nil && loc[0] <= backRefHeadChars
|
||||
}
|
||||
|
||||
// bareAbovePointer reports whether the terminal is a deictic back-reference and
|
||||
// essentially NOTHING ELSE — the form isWeakFinal may discard outright.
|
||||
//
|
||||
// pointsAbove alone is not enough for that, because a pointer can share a short
|
||||
// terminal with the answer: "That's the chain above. Ship Tuesday." is 37 bytes
|
||||
// and the decision is the four words the pointer is not. Discarding it in
|
||||
// favour of an earlier turn throws away the only thing the user needed. Same
|
||||
// shape as the summary-closer fix — a pointer says where the detail is, it does
|
||||
// not say the text beside it is disposable.
|
||||
//
|
||||
// So the reference's own clause is cut out and whatever remains must be filler:
|
||||
// nothing, punctuation, or a throat-clearing "Done —". Cutting the CLAUSE (from
|
||||
// the end of the previous sentence through the reference) rather than testing
|
||||
// position is also what makes a mixed terminal safe — "Anything above 100
|
||||
// boils. See the note above." keeps its first sentence and is correctly not
|
||||
// bare, and a comparative "above," with an interjection after it keeps the rest
|
||||
// of its own sentence for the same reason.
|
||||
func bareAbovePointer(t string) bool {
|
||||
loc := aboveRefRe.FindStringIndex(t)
|
||||
if loc == nil || loc[0] > backRefHeadChars {
|
||||
return false
|
||||
}
|
||||
start := 0
|
||||
if k := strings.LastIndexAny(t[:loc[0]], ".!?\n"); k >= 0 {
|
||||
start = k + 1
|
||||
}
|
||||
rest := strings.TrimSpace(t[:start]) + " " + strings.TrimSpace(t[loc[1]:])
|
||||
return bareRemainderRe.MatchString(strings.Trim(rest, pointerResidueCutset))
|
||||
}
|
||||
|
||||
// pointerResidueCutset is trimmed from both ends of what survives cutting the
|
||||
// pointer's clause — the brackets, quotes, and punctuation a model wraps a
|
||||
// back-reference in ("(Already answered above.)").
|
||||
const pointerResidueCutset = " \t\r\n.,;:!?()[]{}\"'“”‘’*_-—"
|
||||
|
||||
// bareRemainderRe matches a remainder that carries no answer: empty, or only
|
||||
// the filler a closer opens with. Shares fillerWords with summaryPreface so the
|
||||
// two lists cannot drift.
|
||||
var bareRemainderRe = regexp.MustCompile(`(?i)^(` + fillerWords + `[\s,.!:—-]*)*$`)
|
||||
|
||||
// compressionMarkerRe matches a model announcing that what follows is the
|
||||
// short form of something longer ("Short version: …", "TL;DR: …", "In short,
|
||||
// …"). It is the second half of the deictic summary-closer test: the pointer
|
||||
// says the full answer is elsewhere, and this says the text beside it is a
|
||||
// condensation rather than new reasoning.
|
||||
//
|
||||
// Both halves are required, because a deictic pointer alone does not mean the
|
||||
// terminal is disposable. "Given the analysis above, I recommend option B
|
||||
// because X" opens with a pointer and then states a CONCLUSION the earlier
|
||||
// turn never contained — discarding it in favour of that turn would throw away
|
||||
// the answer. A compression marker is the model telling us the opposite.
|
||||
//
|
||||
// A marker WITHOUT a pointer stays out of scope, as summaryCloserRe's own
|
||||
// comment explains: a user who asked for brevity is answered with exactly that
|
||||
// shape.
|
||||
// The marker must OPEN a sentence. Mid-sentence the same words are ordinary
|
||||
// prose carrying new content — "Given the analysis above, the bottom line is
|
||||
// that we need a different vendor" states a conclusion, it does not announce a
|
||||
// condensation — and treating that as disposable is the very failure the
|
||||
// two-part test exists to prevent.
|
||||
var compressionMarkerRe = regexp.MustCompile(`(?i)(^|[.!?:;—]\s*|\n\s*)(short version|shorter version|short answer|tl;?dr|in short|in brief|in summary|in sum|bottom line|net[- ]net|the gist)\b`)
|
||||
|
||||
// summaryCloserRe matches a terminal turn that OPENS with a bookkeeping
|
||||
// acknowledgment of the citation round — "Citations are logged.", "Sources
|
||||
// cited.", "Logged the citations." — the shape a model produces when it
|
||||
@@ -128,7 +235,11 @@ var backRefRe = regexp.MustCompile(`(?i)(already answered|see above|as (i )?(sai
|
||||
// behavior (fail closed). Assembled from named fragments so the alternations
|
||||
// stay legible and extendable.
|
||||
const (
|
||||
summaryPreface = `((done|all set|ok(ay)?)[\s,.!:—-]+)?` // optional "Done —" style opener
|
||||
// fillerWords are the throat-clearing tokens a closer opens with. Shared
|
||||
// with bareRemainderRe, which has to recognise exactly the same set as
|
||||
// "not answer content".
|
||||
fillerWords = `(done|all set|ok(ay)?)`
|
||||
summaryPreface = `(` + fillerWords + `[\s,.!:—-]+)?` // optional "Done —" style opener
|
||||
summaryNouns = `(citations?|sources?|references?|claims?)`
|
||||
// "all" appears here AND in summaryArticle on purpose: as a quantifier
|
||||
// between noun and verb ("Citations all logged.") and as a determiner
|
||||
@@ -180,6 +291,15 @@ const (
|
||||
// genuine final answer that merely contains "as I said" mid-sentence is
|
||||
// longer than this, so it is never treated as weak.
|
||||
weakFinalMaxChars = 120
|
||||
// backRefHeadChars bounds how far into the terminal a deictic "above" may
|
||||
// sit and still read as pointing OUTSIDE this message (see pointsAbove).
|
||||
// It IS weakFinalMaxChars — the same guard ("there is not enough text
|
||||
// before the reference for it to be pointing at content inside this
|
||||
// turn"), expressed as an offset because a summary closer carries a
|
||||
// compression AFTER the pointer and so is not itself short. Defined by
|
||||
// reference, not by repeating the literal: tuning the weak cap without the
|
||||
// offset following it would split one rule into two.
|
||||
backRefHeadChars = weakFinalMaxChars
|
||||
// recoverMinChars: a prior assistant turn this long is treated as a real
|
||||
// answer regardless of how it opens (the preamble filter is not applied at
|
||||
// this length — see isSubstantiveAnswer).
|
||||
@@ -203,13 +323,23 @@ const (
|
||||
)
|
||||
|
||||
// isWeakFinal reports whether a terminal turn's text fails to stand on its own
|
||||
// as the answer: empty/whitespace, or a short pure back-reference.
|
||||
// as the answer: empty/whitespace, or a short pure back-reference. The length
|
||||
// cap is what keeps a genuine answer that merely contains a back-reference
|
||||
// phrase mid-sentence out of the class; past the cap a deferring terminal is
|
||||
// isSummaryCloser's business, under the stricter dwarf bar.
|
||||
//
|
||||
// The deictic half additionally demands that the pointer be the WHOLE message
|
||||
// (bareAbovePointer), not merely present in it. Both classes now enforce the
|
||||
// same rule from opposite ends: a terminal is only disposable when it carries
|
||||
// no answer of its own — proved here by there being nothing beside the pointer,
|
||||
// and in isSummaryCloser by the model declaring what is beside it a
|
||||
// compression.
|
||||
func isWeakFinal(s string) bool {
|
||||
t := strings.TrimSpace(s)
|
||||
if t == "" {
|
||||
return true
|
||||
}
|
||||
return len(t) <= weakFinalMaxChars && backRefRe.MatchString(t)
|
||||
return len(t) <= weakFinalMaxChars && (backRefRe.MatchString(t) || bareAbovePointer(t))
|
||||
}
|
||||
|
||||
// isCitationsOnly reports whether a terminal turn is essentially just a
|
||||
@@ -238,19 +368,38 @@ func isCitationsOnly(s string) bool {
|
||||
return len(residue) <= len(t)/citationDominatedDivisor
|
||||
}
|
||||
|
||||
// isSummaryCloser reports whether a terminal turn is a bookkeeping closer: it
|
||||
// opens with a complete "citations are logged"-style ack sentence (see
|
||||
// summaryCloserRe) and is short enough that whatever follows the ack can only
|
||||
// be a compression of an earlier, fuller answer. Whether that fuller answer
|
||||
// actually exists is modeSummary's job — the dwarf ratio in
|
||||
// isSubstantiveAnswer keeps a matching closer in place when nothing earlier
|
||||
// clearly outweighs it.
|
||||
// isSummaryCloser reports whether a terminal turn defers to an earlier answer
|
||||
// and is short enough that whatever follows the deferral can only be a
|
||||
// compression of it. Two openers qualify:
|
||||
//
|
||||
// - a complete bookkeeping ack sentence — "Citations are logged." — which is
|
||||
// summaryCloserRe, and carries no answer content of its own;
|
||||
// - a deictic back-reference in the opening PLUS a compression marker —
|
||||
// mort issue #1611's "Done — that's the full chain above. Short version:
|
||||
// …" — which the ack shape alone did not cover.
|
||||
//
|
||||
// The second opener needs both halves. A pointer on its own does not make a
|
||||
// terminal disposable: "Given the analysis above, I recommend option B because
|
||||
// X" points backwards and then states a conclusion the earlier turn never
|
||||
// contained, and replacing it with that turn would discard the answer. The
|
||||
// compression marker is the model saying the opposite — that what follows is
|
||||
// the short form of something it already wrote.
|
||||
//
|
||||
// Whether the fuller answer actually exists is modeSummary's job — the dwarf
|
||||
// ratio in isSubstantiveAnswer keeps a matching closer in place when nothing
|
||||
// earlier clearly outweighs it.
|
||||
//
|
||||
// pointsAbove is used here; its sibling backRefRe is NOT. Those fixed phrases
|
||||
// are matched anywhere in the text, and a 300-byte closer has room for a real
|
||||
// answer that merely mentions "as I said" mid-sentence. pointsAbove is
|
||||
// offset-bounded, so it stays anchored to the opening.
|
||||
func isSummaryCloser(s string) bool {
|
||||
t := strings.TrimSpace(s)
|
||||
if t == "" || len(t) > summaryCloserMaxChars {
|
||||
return false
|
||||
}
|
||||
return summaryCloserRe.MatchString(t)
|
||||
return summaryCloserRe.MatchString(t) ||
|
||||
(pointsAbove(t) && compressionMarkerRe.MatchString(t))
|
||||
}
|
||||
|
||||
// lastSubstantiveAssistantText scans msgs newest→oldest (skipping the terminal
|
||||
|
||||
@@ -26,6 +26,23 @@ func TestIsWeakFinal(t *testing.T) {
|
||||
{"crisp-yes", "Yes.", false},
|
||||
{"crisp-status", "It's down, restarting now.", false},
|
||||
{"long-with-as-i-said", long, false}, // >120 chars: not weak despite the phrase
|
||||
|
||||
// The deictic half of the class, inside the length cap. A pointer only
|
||||
// counts when it is ALL there is — see bareAbovePointer.
|
||||
{"bare-above-pointer", "That's the full chain above.", true},
|
||||
{"above-pointer-in-parens", "(the breakdown is above)", true},
|
||||
{"bare-pointer-with-filler-opener", "Done. See the chain above.", true},
|
||||
|
||||
// A pointer sharing the terminal with real content is NOT weak: the
|
||||
// content beside it is the answer, and discarding the terminal would
|
||||
// throw it away.
|
||||
{"pointer-plus-a-decision", "That's the chain above. Ship Tuesday.", false},
|
||||
{"pointer-plus-a-choice", "See the summary above. Option B wins.", false},
|
||||
{"pointer-mid-sentence-then-answer", "As shown above, the answer is sixty minutes.", false},
|
||||
{"prepositional-then-deictic-with-content", "Anything above 100 boils. See the note above.", false},
|
||||
{"comparative-with-an-interjection", "Anything above, say, 40 degrees is a problem for the pump.", false},
|
||||
{"prepositional-above-not-weak", "Anything above 100 degrees boils off.", false},
|
||||
{"1611-closer-too-long-for-weak", closer1611, false}, // 220 bytes: isSummaryCloser's job
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
@@ -36,6 +53,91 @@ func TestIsWeakFinal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// closer1611 is the verbatim terminal turn from mort run
|
||||
// 8eea3e82-b61b-4174-9750-9aa2f4bde4d4 (issue #1611): the model front-loaded a
|
||||
// 2,245-char analysis into the cite-call turn and closed with this 220-byte
|
||||
// pointer-plus-compression. Too long for the weak-final cap and carrying no
|
||||
// citations ack, it matched none of the three original shapes and was
|
||||
// delivered verbatim — the user saw a summary referring to a "chain above"
|
||||
// that had never been posted.
|
||||
const closer1611 = "Done — that's the full chain above. Short version: it's not one incident, it's the confluence of the Iran war, the Epstein files, and three ex-allies now openly plotting a third-party movement that finally set him off."
|
||||
|
||||
// analysis1611 stands in for that run's front-loaded analysis: long enough to
|
||||
// dwarf closer1611 (>3x its 220 bytes). Shared by the finalOutput table and
|
||||
// the end-to-end Run test so the two cannot drift apart.
|
||||
func analysis1611() string {
|
||||
return strings.TrimSpace(strings.Repeat("The break was the Iran strikes, then the Epstein files. ", 12))
|
||||
}
|
||||
|
||||
func TestPointsAbove(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
// Deictic: "above" ends its clause, so it points at earlier text.
|
||||
{"1611-verbatim", closer1611, true},
|
||||
{"sentence-final", "That's the full chain above.", true},
|
||||
{"comma", "As shown above, the answer is 60 minutes.", true},
|
||||
{"end-of-string", "The full breakdown is above", true},
|
||||
{"line-final", "Everything is above\n\nShort version: yes.", true},
|
||||
{"line-final-crlf", "Everything is above\r\nShort version: yes.", true},
|
||||
{"crlf-at-end", "The full breakdown is above\r\n", true},
|
||||
{"closing-paren", "(the detail is above).", true},
|
||||
{"semicolon", "It's above; the short answer is no.", true},
|
||||
|
||||
// Prepositional: "above" continues into a noun phrase.
|
||||
{"above-a-number", "Anything above 100 degrees boils off.", false},
|
||||
{"above-the-fold", "The banner sits above the fold on every page.", false},
|
||||
{"above-all", "Above all, keep the deploy green.", false},
|
||||
{"above-average", "Turnout was above average in three counties.", false},
|
||||
|
||||
// Hyphenated compounds. \b holds between "above" and "-", so a literal
|
||||
// '-' in the terminator class makes all of these read as deictic. The
|
||||
// space-separated cases above do NOT cover this: they are a different
|
||||
// character, and one passed while the other was broken.
|
||||
{"hyphen-above-average", "Turnout was above-average in three counties.", false},
|
||||
{"hyphen-above-board", "The deal was above-board from the start.", false},
|
||||
{"hyphen-above-ground", "Run the above-ground cable along the fence.", false},
|
||||
{"hyphen-above-mentioned", "The above-mentioned findings are attached.", false},
|
||||
|
||||
{"no-above-at-all", "42", false},
|
||||
{"empty", "", false},
|
||||
|
||||
// Offset bound: past backRefHeadChars there IS enough text before the
|
||||
// reference for it to be pointing inside this same message. The
|
||||
// lengths are LITERALS, not backRefHeadChars +/- n: a case sized from
|
||||
// the constant it is meant to pin moves with it, and a break-check
|
||||
// that widened the bound to 100000 sailed straight through.
|
||||
{"late-reference-not-a-pointer", strings.Repeat("x", 200) + " as shown above.", false},
|
||||
// The exact boundary, both sides: "above" starts at index 120 (allowed,
|
||||
// the bound is <=) and at 121 (rejected). 119/120 x's plus the space
|
||||
// puts the 'a' on 120/121 — the separator is needed because \b will not
|
||||
// hold between "x" and "above".
|
||||
{"reference-exactly-at-the-bound", strings.Repeat("x", 119) + " above.", true},
|
||||
{"reference-one-past-the-bound", strings.Repeat("x", 120) + " above.", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := pointsAbove(c.in); got != c.want {
|
||||
t.Errorf("pointsAbove(%q) = %v, want %v", c.in, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// The two bounds are ONE guard expressed two ways (see backRefHeadChars).
|
||||
// Pinned here so decoupling them is a test failure, not a silent drift.
|
||||
if backRefHeadChars != weakFinalMaxChars {
|
||||
t.Errorf("backRefHeadChars = %d, weakFinalMaxChars = %d: the offset bound and the "+
|
||||
"weak-final cap are the same guard and must stay equal",
|
||||
backRefHeadChars, weakFinalMaxChars)
|
||||
}
|
||||
if weakFinalMaxChars != 120 {
|
||||
t.Errorf("weakFinalMaxChars = %d, want 120: the literal-length cases in this table "+
|
||||
"pin the bound at 120 and must be resized with it", weakFinalMaxChars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsCitationsOnly(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
@@ -102,6 +204,27 @@ func TestIsSummaryCloser(t *testing.T) {
|
||||
{"mentions-citations-midsentence", "The paper's citations are what got it retracted.", false},
|
||||
{"crisp-number", "42", false},
|
||||
{"over-cap", "Citations are logged. " + strings.Repeat("The long version has many more details worth keeping. ", 6), false}, // >300: too substantial to replace
|
||||
|
||||
// A deictic back-reference opener also qualifies — the #1611 shape,
|
||||
// which carries no citations ack at all.
|
||||
{"1611-verbatim", closer1611, true},
|
||||
{"above-pointer-plus-tldr", "That's the whole picture above. In short: the merger fell through.", true},
|
||||
{"prepositional-above-is-not-a-closer", "Anything above 100 degrees boils off, which is why the sample evaporated.", false},
|
||||
|
||||
// The compression marker has to OPEN a sentence. Mid-sentence the same
|
||||
// words are prose carrying NEW content, not an announcement that what
|
||||
// follows is a condensation.
|
||||
{"marker-mid-sentence-is-new-content", "Given the analysis above, the bottom line is that we need a different vendor entirely.", false},
|
||||
{"marker-mid-sentence-in-short", "That is the chain above, and in short supply of alternatives we went with B.", false},
|
||||
{"marker-opening-after-a-colon", "That's the chain above: in short, the merger fell through.", true},
|
||||
|
||||
// A deictic pointer WITHOUT a compression marker is not a summary
|
||||
// closer: these state a conclusion the earlier turn never contained,
|
||||
// so replacing them with that turn would discard the answer.
|
||||
{"pointer-then-a-recommendation", "Given the analysis above, I recommend option B: it is the only one that survives a regional outage.", false},
|
||||
{"pointer-then-a-decision", "Based on everything above, we should ship Tuesday and hold the migration until the following sprint.", false},
|
||||
{"pointer-then-a-new-caveat", "That is the chain above. One thing it misses: the Senate vote is scheduled before any of this takes effect.", false},
|
||||
{"1611-over-cap", closer1611 + " " + strings.Repeat("Plenty more detail worth keeping here. ", 4), false}, // >300
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
@@ -147,6 +270,25 @@ func TestFinalOutput(t *testing.T) {
|
||||
// Matches BOTH the summary ack and backRefRe, within the 120-byte weak cap,
|
||||
// and long enough (>~92 bytes) that longAnswer would fail the summary bar.
|
||||
bothMatchCloser := "Citations are logged. As I mentioned above, the full detail on the money sources is in my earlier message."
|
||||
// The #1611 pair: a front-loaded analysis that dwarfs its 220-byte closer.
|
||||
analysis := analysis1611()
|
||||
// A 96-byte BARE deictic closer: nothing but filler and the pointer's own
|
||||
// clause, inside the weak-final cap (120), and sized so the two bars
|
||||
// actually DISAGREE — 3x96 = 288 > longAnswer's 275, so the summary
|
||||
// closer's dwarf ratio would reject longAnswer while the modeBackRef bar
|
||||
// (>=200 bytes, no ratio) accepts it. A shorter closer would pass under
|
||||
// either bar and prove nothing. It must also stay BARE: an earlier draft
|
||||
// ended "…so there is no point repeating it", and prose after the
|
||||
// reference is indistinguishable from an answer, so bareAbovePointer
|
||||
// correctly stopped treating it as disposable.
|
||||
shortAbovePointer := "Done — that is the complete chain, start to finish, exactly as I worked it out for you, above."
|
||||
// A deictic pointer followed by a NEW conclusion (no compression marker):
|
||||
// >120 bytes so isWeakFinal cannot claim it, and it must not be treated as
|
||||
// a summary closer either.
|
||||
pointerThenConclusion := "Given the analysis above, I recommend option B: it is the only one that survives a regional outage without a manual failover step."
|
||||
// A terminal using "above" as a PREPOSITION — not a back-reference, so it
|
||||
// must survive verbatim next to a dwarfing prior turn.
|
||||
prepositionalTerminal := "Anything above 100 degrees boils off, which is exactly why the sample evaporated overnight in the unsealed tray."
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -387,6 +529,99 @@ func TestFinalOutput(t *testing.T) {
|
||||
terminal: b3cb9ee9Closer,
|
||||
want: hugeAnswer,
|
||||
},
|
||||
{
|
||||
// mort issue #1611: a pointer-plus-compression closer with no
|
||||
// citations ack. The 2,245-char analysis was front-loaded into the
|
||||
// cite turn; the closer pointed at a "chain above" the user never
|
||||
// saw. Recover the analysis and discard the closer.
|
||||
name: "above-pointer closer discarded when the front-loaded answer dwarfs it",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("what set off the Truth Social rant?"),
|
||||
asst(analysis, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(closer1611),
|
||||
},
|
||||
terminal: closer1611,
|
||||
want: analysis,
|
||||
},
|
||||
{
|
||||
// The dwarf ratio governs the new opener too: a prior turn that is
|
||||
// longer but not clearly the fuller original (here ~275 bytes vs a
|
||||
// 220-byte closer, under 3x) must not displace a closer that
|
||||
// carries real answer content.
|
||||
name: "above-pointer closer kept when the prior turn does not dwarf it",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(closer1611),
|
||||
},
|
||||
terminal: closer1611,
|
||||
want: closer1611,
|
||||
},
|
||||
{
|
||||
// A prepositional "above" is not a back-reference: this terminal
|
||||
// stands on its own and must be returned verbatim even though a
|
||||
// much longer prior turn exists.
|
||||
name: "prepositional above is not hijacked by a longer prior turn",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("why did the sample evaporate?"),
|
||||
asst(hugeAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(prepositionalTerminal),
|
||||
},
|
||||
terminal: prepositionalTerminal,
|
||||
want: prepositionalTerminal,
|
||||
},
|
||||
{
|
||||
// A SHORT deictic closer takes the modeBackRef bar, not the summary
|
||||
// closer's mandatory dwarf ratio — even though it carries a scrap
|
||||
// of answer content ("Done."). Deliberate, and the same contract a
|
||||
// short "see above" closer has always had: within the 120-byte cap
|
||||
// there is no room for both a pointer and a real answer, so a
|
||||
// >=200-byte prior turn wins without having to be 3x. Here the
|
||||
// ratio would demand ~330 bytes and wrongly keep the pointer.
|
||||
// (gadfly/sonnet flagged the asymmetry; this pins it.)
|
||||
name: "short above-pointer closer uses the back-ref bar, not the dwarf ratio",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("q?"),
|
||||
asst(longAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(shortAbovePointer),
|
||||
},
|
||||
terminal: shortAbovePointer,
|
||||
want: longAnswer,
|
||||
},
|
||||
{
|
||||
// The isWeakFinal twin of the case below: a SHORT pointer that
|
||||
// shares its terminal with the decision. Before bareAbovePointer
|
||||
// this was weak, so a >=200-byte prior turn replaced it and "Ship
|
||||
// Tuesday" — the only thing the user needed — was discarded.
|
||||
name: "short above-pointer sharing the terminal with the answer is not discarded",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("when do we ship?"),
|
||||
asst(hugeAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst("That's the chain above. Ship Tuesday."),
|
||||
},
|
||||
terminal: "That's the chain above. Ship Tuesday.",
|
||||
want: "That's the chain above. Ship Tuesday.",
|
||||
},
|
||||
{
|
||||
// A pointer plus a NEW conclusion is not a compression, so it must
|
||||
// survive verbatim even though a much longer prior turn exists —
|
||||
// otherwise the recommendation is thrown away in favour of the
|
||||
// analysis it was drawn from.
|
||||
name: "above-pointer closer with a new conclusion is not discarded",
|
||||
msgs: []llm.Message{
|
||||
llm.UserText("which option?"),
|
||||
asst(hugeAnswer, cite...),
|
||||
llm.ToolResultsMessage(llm.ToolResult{ID: "c1", Name: "cite", Content: "ok"}),
|
||||
asst(pointerThenConclusion),
|
||||
},
|
||||
terminal: pointerThenConclusion,
|
||||
want: pointerThenConclusion,
|
||||
},
|
||||
{
|
||||
// A closer matching BOTH the ack shape and a back-reference
|
||||
// carries no answer content, so the back-ref test must win and the
|
||||
@@ -536,3 +771,34 @@ func TestRun_RecoversFrontLoadedAnswerWithCitations(t *testing.T) {
|
||||
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRun_RecoversFrontLoadedAnswerOverAboveRefCloser reproduces mort issue
|
||||
// #1611 end-to-end: the model front-loads its analysis into the cite-call turn
|
||||
// and closes with a pointer at that invisible text plus a one-line
|
||||
// compression. The delivered output must be the front-loaded analysis, with no
|
||||
// extra model call.
|
||||
func TestRun_RecoversFrontLoadedAnswerOverAboveRefCloser(t *testing.T) {
|
||||
analysis := analysis1611()
|
||||
fp := fake.New("fp")
|
||||
fp.Enqueue("test-model",
|
||||
fake.ReplyWith(llm.Response{
|
||||
Parts: []llm.Part{llm.Text(analysis)},
|
||||
ToolCalls: []llm.ToolCall{{ID: "c1", Name: "cite", Arguments: json.RawMessage(`{}`)}},
|
||||
FinishReason: llm.FinishToolCalls,
|
||||
Usage: llm.Usage{InputTokens: 10, OutputTokens: 5},
|
||||
}),
|
||||
fake.Reply(closer1611),
|
||||
)
|
||||
|
||||
a := New(newModel(t, fp), "sys", WithToolbox(citeToolbox(t)))
|
||||
res, err := a.Run(context.Background(), "what set off the Truth Social rant?")
|
||||
if err != nil {
|
||||
t.Fatalf("Run: %v", err)
|
||||
}
|
||||
if res.Output != analysis {
|
||||
t.Errorf("Output = %q, want recovered front-loaded analysis %q", res.Output, analysis)
|
||||
}
|
||||
if n := len(fp.Calls()); n != 2 {
|
||||
t.Errorf("model calls = %d, want 2 (no extra nudge turn)", n)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user