Files
pansy/internal/agent/runtime_test.go
T
steveandClaude Fable 5 d4eb62a2ba
Build image / build-and-push (push) Successful in 8s
Address #132 review: one verb list, self-reporting tools, rune-safe log
- changeClaim is built from one changeVerbs list; the opener is just
  done/fixed/undone so an informational "Updated totals:" can't trip it.
- public_link (get reads) and undo_change (nothing left to revert) are
  self-reporting: their success no longer counts as a change by name; the
  adapter says whether they changed something (noteChange / didChange).
- whenMissing covers the object and plant tools too (move/update/delete
  object, clear/remove plantings by object, update/delete plant).
- The step summary cuts on a rune boundary.

Co-Authored-By: Claude Fable 5 <[email protected]>
2026-08-23 02:49:55 -04:00

672 lines
27 KiB
Go

package agent
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"unicode/utf8"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
"gitea.stevedudenhoeffer.com/steve/pansy/internal/domain"
"gitea.stevedudenhoeffer.com/steve/pansy/internal/service"
)
// scriptedRunner wires a Runner to a fake provider so the whole loop — tools,
// change-set scoping, loop guards — is exercised with no live model.
func scriptedRunner(t *testing.T, svc *service.Service, steps ...fake.Step) *Runner {
t.Helper()
p := fake.New("fake")
for _, s := range steps {
p.Enqueue("m", s)
}
model, err := p.Model("m")
if err != nil {
t.Fatalf("fake model: %v", err)
}
return &Runner{svc: svc, model: model}
}
// toolCall scripts one model turn that calls a tool.
func toolCall(name string, args any) fake.Step {
raw, _ := json.Marshal(args)
return fake.ReplyWith(llm.Response{
FinishReason: llm.FinishToolCalls,
ToolCalls: []llm.ToolCall{{ID: "c1", Name: name, Arguments: raw}},
})
}
// TestTurnIsOneChangeSet is the acceptance criterion the whole "act freely"
// posture rests on: a turn that clears a bed and replants it — one object edit
// and many planting inserts — has to undo as ONE action, not thirteen.
func TestTurnIsOneChangeSet(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
garlic := mustPlant(t, svc, owner, "Garlic", 15, "🧄")
cucumber := mustPlant(t, svc, owner, "Cucumber", 45, "🥒")
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{
Kind: domain.KindBed, Name: "Garlic bed", XCM: 1000, YCM: 1000, WidthCM: 400, HeightCM: 400,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
if _, err := svc.FillNamedRegion(ctx, owner, bed.ID, "all", garlic.ID, nil, service.FillClump, nil); err != nil {
t.Fatalf("seed garlic: %v", err)
}
before, _, err := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if err != nil {
t.Fatalf("history: %v", err)
}
r := scriptedRunner(t, svc,
toolCall("clear_object", map[string]any{"objectId": bed.ID}),
toolCall("fill_region", map[string]any{"objectId": bed.ID, "region": "all", "plantId": cucumber.ID}),
fake.Reply("Cleared the garlic and replanted the bed with cucumbers."),
)
turn, err := r.Run(ctx, owner, g.ID, "change the garlic bed to cucumbers this year", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
if turn.ChangeSetID == nil {
t.Fatal("a turn that changed things produced no change set")
}
if !strings.Contains(turn.Reply, "cucumbers") {
t.Errorf("reply = %q", turn.Reply)
}
// Exactly ONE new change set, whatever the model did inside the turn.
after, _, err := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if err != nil {
t.Fatalf("history: %v", err)
}
if len(after)-len(before) != 1 {
t.Fatalf("turn produced %d change sets, want exactly 1", len(after)-len(before))
}
cs := after[0]
if cs.Source != domain.SourceAgent {
t.Errorf("source = %q, want agent", cs.Source)
}
if cs.Summary != "change the garlic bed to cucumbers this year" {
t.Errorf("summary = %q, want the user's own words", cs.Summary)
}
// The bed really is cucumbers now.
full, _ := svc.GardenFull(ctx, owner, g.ID, nil)
if len(full.Plantings) == 0 {
t.Fatal("bed ended up empty")
}
for _, p := range full.Plantings {
if p.PlantID != cucumber.ID {
t.Errorf("unexpected plant %d still in the bed", p.PlantID)
}
}
// And one undo puts it back.
if _, conflicts, err := svc.RevertChangeSet(ctx, owner, *turn.ChangeSetID, domain.SourceUI); err != nil || len(conflicts) != 0 {
t.Fatalf("undo: err=%v conflicts=%+v", err, conflicts)
}
full, _ = svc.GardenFull(ctx, owner, g.ID, nil)
for _, p := range full.Plantings {
if p.PlantID != garlic.ID {
t.Errorf("after undo the bed holds plant %d, want the garlic back", p.PlantID)
}
}
}
// TestViewerGetsAnExplainableRefusal — the ACL story only works if the model can
// narrate the refusal, so a tool denial has to reach it as a tool RESULT it can
// read, not as a 500 that ends the run.
func TestViewerGetsAnExplainableRefusal(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
viewer, err := svc.Register(ctx, service.RegisterInput{Email: "[email protected]", DisplayName: "V", Password: "password123"})
if err != nil {
t.Fatalf("register: %v", err)
}
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{
Kind: domain.KindBed, XCM: 1000, YCM: 1000, WidthCM: 400, HeightCM: 400,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
if _, err := svc.AddShare(ctx, owner, g.ID, "[email protected]", domain.RoleViewer); err != nil {
t.Fatalf("share: %v", err)
}
// A viewer can't open a change set at all, so the turn is refused up front —
// before any model call — and the API turns that into a plain explanation.
r := scriptedRunner(t, svc, fake.Reply("unused"))
_, err = r.Run(ctx, viewer.ID, g.ID, "plant garlic in that bed", "", nil, nil)
if !errors.Is(err, domain.ErrForbidden) {
t.Fatalf("viewer turn err = %v, want ErrForbidden", err)
}
// And at the tool layer, a refusal comes back as a readable tool result
// rather than killing the run.
box := NewToolbox(svc, viewer.ID, "")
raw, _ := json.Marshal(map[string]any{"objectId": bed.ID})
res := box.Execute(ctx, llm.ToolCall{ID: "1", Name: "clear_object", Arguments: raw})
if !res.IsError {
t.Fatal("a viewer's clear_object succeeded")
}
if res.Content == "" {
t.Error("the refusal carried no text for the model to explain")
}
}
// TestRunStopsAtTheStepCap — a model that gets wedged should stop on its own,
// and what it managed to do must still be recorded and undoable.
func TestRunStopsAtTheStepCap(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
// More describe_garden calls than the cap allows, forever.
steps := make([]fake.Step, 0, maxSteps+4)
for i := 0; i < maxSteps+4; i++ {
steps = append(steps, toolCall("describe_garden", map[string]any{"gardenId": g.ID}))
}
r := scriptedRunner(t, svc, steps...)
turn, err := r.Run(ctx, owner, g.ID, "look at the garden", "", nil, nil)
if err != nil {
t.Fatalf("a capped run should end cleanly, got %v", err)
}
if !turn.Truncated {
t.Error("turn.Truncated = false; the run hit the cap and should say so")
}
if turn.Reply == "" {
t.Error("a capped run said nothing; silence reads as a hang")
}
// It only read, so there is nothing to undo — and no empty change set either.
if turn.ChangeSetID != nil {
t.Errorf("a read-only turn produced change set %d", *turn.ChangeSetID)
}
}
// TestReadOnlyTurnWritesNoChangeSet — asking a question must not litter the
// history with empty entries.
func TestReadOnlyTurnWritesNoChangeSet(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
before, _, _ := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
r := scriptedRunner(t, svc,
toolCall("describe_garden", map[string]any{"gardenId": g.ID}),
fake.Reply("It's empty — nothing planted yet."),
)
turn, err := r.Run(ctx, owner, g.ID, "what's in the garden?", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
if turn.ChangeSetID != nil {
t.Errorf("a question produced change set %d", *turn.ChangeSetID)
}
after, _, _ := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if len(after) != len(before) {
t.Errorf("history grew by %d for a read-only turn", len(after)-len(before))
}
}
// TestNewRunnerNeedsConfiguration — an instance with no key or no model must not
// get a half-built runner; the caller treats the error as "no assistant" and
// carries on. (Whether the assistant is ENABLED is resolved before NewRunner is
// reached, so it isn't NewRunner's concern any more.)
func TestNewRunnerNeedsConfiguration(t *testing.T) {
svc, _ := newAgentTestService(t)
for _, tc := range []struct{ key, model string }{
{"", "ollama-cloud/x"},
{"k", ""},
{"k", " "},
} {
if _, err := NewRunner(svc, tc.key, tc.model); err == nil {
t.Errorf("NewRunner accepted key=%q model=%q", tc.key, tc.model)
}
}
// A model spec naming a provider that doesn't exist is a configuration
// error, not a panic at first use.
if _, err := NewRunner(svc, "k", "nonesuch/model"); err == nil {
t.Error("NewRunner accepted an unresolvable model spec")
}
}
// TestTurnSummaryFitsAHistoryRow — the user's own words are the most useful
// label for a change set, but a paragraph would wreck the list.
func TestTurnSummaryFitsAHistoryRow(t *testing.T) {
if got := turnSummary(" change the garlic bed\n to cucumbers "); got != "change the garlic bed to cucumbers" {
t.Errorf("turnSummary = %q", got)
}
long := turnSummary(strings.Repeat("plant garlic ", 40))
if len(long) > 130 || !strings.HasSuffix(long, "…") {
t.Errorf("long summary = %q (%d chars)", long, len(long))
}
}
// TestSystemPromptStatesTheCompassConvention — -y being north is not guessable,
// and a model that assumes otherwise plants the wrong end of the bed.
func TestSystemPromptStatesTheCompassConvention(t *testing.T) {
p := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 500, HeightCM: 400, UnitPref: domain.UnitImperial}, "2026-08-22")
for _, want := range []string{"NORTH", "-y", "CENTIMETERS", "Plot", "version"} {
if !strings.Contains(p, want) {
t.Errorf("system prompt is missing %q:\n%s", want, p)
}
}
if !strings.Contains(p, fmt.Sprintf("%.0f", 500.0)) {
t.Error("system prompt doesn't state the garden's size")
}
}
// TestPartialWorkSurvivesATimeout is the finding that mattered most on this PR.
//
// The run context carries a timeout. When it fires, WithChangeSet's recovery
// path has to record what already committed — and doing that with the SAME
// dead context would fail, losing the history for changes that really happened.
// The user-facing message says "anything I'd already changed is in History", so
// this isn't just a gap, it's a promise the code has to keep.
func TestPartialWorkSurvivesATimeout(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{
Kind: domain.KindBed, Name: "Bed", XCM: 1000, YCM: 1000, WidthCM: 400, HeightCM: 400,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
before, _, _ := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
// A turn that renames the bed, then dies with the context already cancelled.
cancelled, cancel := context.WithCancel(ctx)
r := scriptedRunner(t, svc,
toolCall("move_object", map[string]any{
"objectId": bed.ID, "xCm": 600.0, "yCm": 600.0, "version": bed.Version,
}),
fake.Step{Err: context.DeadlineExceeded},
)
// Cancel once the first tool call has landed, so the failure path runs with a
// dead context — exactly the timeout case.
go func() {
time.Sleep(50 * time.Millisecond)
cancel()
}()
_, err = r.Run(cancelled, owner, g.ID, "move the bed", "", nil, nil)
if err == nil {
t.Fatal("expected the turn to fail")
}
// The move committed, so it must be in history and undoable.
after, _, herr := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if herr != nil {
t.Fatalf("history: %v", herr)
}
if len(after) != len(before)+1 {
t.Fatalf("the failed turn recorded %d change sets, want 1 — its work is otherwise un-undoable",
len(after)-len(before))
}
if !strings.Contains(after[0].Summary, "failed partway") {
t.Errorf("summary = %q, want it marked as partial", after[0].Summary)
}
if _, conflicts, rerr := svc.RevertChangeSet(ctx, owner, after[0].ID, domain.SourceUI); rerr != nil || len(conflicts) != 0 {
t.Fatalf("the partial turn should be undoable: err=%v conflicts=%+v", rerr, conflicts)
}
o, _ := svc.DescribeGarden(ctx, owner, g.ID, nil)
if len(o.Objects) > 0 && o.Objects[0].XCM != bed.XCM {
t.Errorf("undo left the bed at %v, want %v", o.Objects[0].XCM, bed.XCM)
}
}
// TestTurnSummaryTrimsByRunes — slicing a byte offset would cut a multibyte
// character in half and store invalid UTF-8 in the history summary.
func TestTurnSummaryTrimsByRunes(t *testing.T) {
// 200 multibyte runes: a byte slice at 120 would land mid-character.
got := turnSummary(strings.Repeat("🌱", 200))
if !utf8.ValidString(got) {
t.Errorf("turnSummary produced invalid UTF-8: %q", got)
}
if !strings.HasSuffix(got, "…") {
t.Errorf("long summary should be elided, got %q", got)
}
if n := utf8.RuneCountInString(got); n > 121 {
t.Errorf("summary is %d runes, want it trimmed", n)
}
}
// TestSystemPromptKnowsTheDayAndTheGardenersUnits — two things the live model
// got wrong for want of being told: it dated journal entries with the year it
// remembered from training, and answered a feet-and-inches gardener in
// centimeters. The conduct rules are checked by their load-bearing phrases.
func TestSystemPromptKnowsTheDayAndTheGardenersUnits(t *testing.T) {
imperial := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 731.52, HeightCM: 731.52, UnitPref: domain.UnitImperial}, "2026-08-22")
for _, want := range []string{
"Today is 2026-08-22",
"feet and inches",
"24.0 x 24.0 ft",
"never describe a change you did not make",
"undo_change",
"Undo this",
"rather than clearing beds on a guess",
"not to narrate your own planting",
`"Plot — <year>"`,
"remove_plantings",
"move_planting",
"list_plantings",
} {
if !strings.Contains(imperial, want) {
t.Errorf("imperial prompt is missing %q", want)
}
}
metric := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 500, HeightCM: 400, UnitPref: domain.UnitMetric}, "2026-08-22")
if strings.Contains(metric, "feet and inches") {
t.Error("metric prompt tells the model to answer in feet and inches")
}
if !strings.Contains(metric, "500 x 400 cm") {
t.Error("metric prompt doesn't state the garden's size in cm")
}
}
// TestRunRejectsAMalformedToday — the date reaches every tool as a default, so a
// bad one must stop the turn before the model runs, not fail its first fill.
func TestRunRejectsAMalformedToday(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
r := scriptedRunner(t, svc, fake.Reply("unused"))
if _, err := r.Run(ctx, owner, g.ID, "hello", "yesterday", nil, nil); !errors.Is(err, domain.ErrInvalidInput) {
t.Errorf("Run with today=%q: err = %v, want ErrInvalidInput", "yesterday", err)
}
}
// TestTurnDatesItsWorkTheGardenersDay — what a turn plants is dated the day the
// gardener sent it, not the server's UTC day (which is tomorrow by nine in the
// evening in Ohio) and not a day the model chose.
func TestTurnDatesItsWorkTheGardenersDay(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
garlic := mustPlant(t, svc, owner, "Garlic", 15, "🧄")
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{
Kind: domain.KindBed, Name: "Bed", XCM: 1000, YCM: 1000, WidthCM: 200, HeightCM: 200,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
r := scriptedRunner(t, svc,
toolCall("fill_region", map[string]any{"objectId": bed.ID, "region": "all", "plantId": garlic.ID}),
fake.Reply("Filled the bed with garlic."),
)
if _, err := r.Run(ctx, owner, g.ID, "fill the bed with garlic", "2026-08-22", nil, nil); err != nil {
t.Fatalf("Run: %v", err)
}
full, err := svc.GardenFull(ctx, owner, g.ID, nil)
if err != nil {
t.Fatalf("GardenFull: %v", err)
}
if len(full.Plantings) == 0 {
t.Fatal("the turn planted nothing")
}
for _, p := range full.Plantings {
if p.PlantedAt == nil || *p.PlantedAt != "2026-08-22" {
t.Errorf("plop %d plantedAt = %v, want the gardener's day 2026-08-22", p.ID, p.PlantedAt)
}
}
}
// TestTurnOnAnotherGardenFilesHistoryThere — a turn is scoped to one garden, but
// nothing stops the model from pointing a tool at an object in another garden
// the person can edit ("do the same in my other garden"). Those revisions must
// land in THAT garden's history, as the agent's work, where its undo can see
// them — not in the open scope, where undoing this turn would quietly revert
// rows in a garden the person isn't looking at.
func TestTurnOnAnotherGardenFilesHistoryThere(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
a, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "A", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden A: %v", err)
}
b, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "B", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden B: %v", err)
}
bedB, err := svc.CreateObject(ctx, owner, b.ID, service.ObjectInput{
Kind: domain.KindBed, Name: "Bed", XCM: 1000, YCM: 1000, WidthCM: 200, HeightCM: 200,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
beforeA, _, _ := svc.GardenHistory(ctx, owner, a.ID, 0, 0)
beforeB, _, _ := svc.GardenHistory(ctx, owner, b.ID, 0, 0)
r := scriptedRunner(t, svc,
toolCall("update_object", map[string]any{"objectId": bedB.ID, "version": bedB.Version, "name": "Renamed from A"}),
fake.Reply("Renamed the bed in B."),
)
turn, err := r.Run(ctx, owner, a.ID, "rename the bed in my other garden", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
if turn.ChangeSetID != nil {
t.Errorf("the turn on A produced change set %d, but it changed nothing in A", *turn.ChangeSetID)
}
afterA, _, _ := svc.GardenHistory(ctx, owner, a.ID, 0, 0)
if len(afterA) != len(beforeA) {
t.Errorf("A's history grew by %d for a change made in B", len(afterA)-len(beforeA))
}
afterB, _, _ := svc.GardenHistory(ctx, owner, b.ID, 0, 0)
if len(afterB) != len(beforeB)+1 {
t.Fatalf("B's history grew by %d, want 1", len(afterB)-len(beforeB))
}
if cs := afterB[0]; cs.Source != domain.SourceAgent || cs.AgentRunID == nil {
t.Errorf("B's entry = source %q, run %v; want the agent's, with its run id", cs.Source, cs.AgentRunID)
}
// And it undoes from B, where the person would look for it.
if _, conflicts, err := svc.RevertChangeSet(ctx, owner, afterB[0].ID, domain.SourceUI); err != nil || len(conflicts) != 0 {
t.Fatalf("undo from B: err=%v conflicts=%+v", err, conflicts)
}
d, _ := svc.DescribeGarden(ctx, owner, b.ID, nil)
if len(d.Objects) != 1 || d.Objects[0].Name != "Bed" {
t.Errorf("after undo B's bed is %+v, want its original name back", d.Objects)
}
}
// TestTurnThatOnlyUndoesIsItselfUndoable — a revert is its own change set,
// outside the turn's scope, so a turn that did nothing but undo would come back
// with no change of its own; the reply would then be the one change in the
// conversation without an "Undo this". It gets the revert instead — a redo.
func TestTurnThatOnlyUndoesIsItselfUndoable(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
beets := mustPlant(t, svc, owner, "Beets", 10, "🫜")
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{Kind: domain.KindBed, Name: "Bed", XCM: 1000, YCM: 1000, WidthCM: 400, HeightCM: 400})
if err != nil {
t.Fatalf("bed: %v", err)
}
// The beets went in by hand in the editor: the change the person wants undone.
if _, err := svc.FillNamedRegion(ctx, owner, bed.ID, "all", beets.ID, nil, service.FillClump, nil); err != nil {
t.Fatalf("plant beets: %v", err)
}
history, _, err := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if err != nil || len(history) == 0 {
t.Fatalf("history: %v (%d entries)", err, len(history))
}
planted := history[0]
r := scriptedRunner(t, svc,
toolCall("read_history", map[string]any{"gardenId": g.ID}),
toolCall("undo_change", map[string]any{"changeSetId": planted.ID}),
fake.Reply("Undone — the beets are out of the bed again."),
)
turn, err := r.Run(ctx, owner, g.ID, "undo the beets", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
full, err := svc.GardenFull(ctx, owner, g.ID, nil)
if err != nil {
t.Fatalf("GardenFull: %v", err)
}
if len(full.Plantings) != 0 {
t.Fatalf("%d beets still in the bed after the undo", len(full.Plantings))
}
after, _, _ := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if len(after) != len(history)+1 {
t.Fatalf("history grew by %d, want exactly the revert", len(after)-len(history))
}
revert := after[0]
if revert.Source != domain.SourceAgent || revert.RevertsID == nil || *revert.RevertsID != planted.ID {
t.Errorf("newest entry = %+v; want the agent's revert of %d", revert, planted.ID)
}
if turn.ChangeSetID == nil || *turn.ChangeSetID != revert.ID {
t.Fatalf("turn.ChangeSetID = %v, want the revert %d so the reply can offer a redo", turn.ChangeSetID, revert.ID)
}
// And "Undo this" on that reply is a redo.
if _, conflicts, err := svc.RevertChangeSet(ctx, owner, *turn.ChangeSetID, domain.SourceUI); err != nil || len(conflicts) != 0 {
t.Fatalf("redo: err=%v conflicts=%+v", err, conflicts)
}
full, _ = svc.GardenFull(ctx, owner, g.ID, nil)
if len(full.Plantings) == 0 {
t.Error("redoing the turn did not put the beets back")
}
}
// TestSystemPromptCarriesTheGardenersNotes — the notes are the assistant's
// memory: what the gardener told it about the place comes back on every turn,
// quoted as their words rather than pasted as instructions.
func TestSystemPromptCarriesTheGardenersNotes(t *testing.T) {
with := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 500, HeightCM: 400, UnitPref: domain.UnitMetric,
Notes: "Zone 6a.\nLast frost \"usually\" May 10."}, "2026-08-23")
for _, want := range []string{
`"Zone 6a.\nLast frost \"usually\" May 10."`,
"update_garden",
"undo_change",
"describe_garden with a year",
"readyAround",
"create_garden",
"confirmed=true",
} {
if !strings.Contains(with, want) {
t.Errorf("prompt is missing %q", want)
}
}
if strings.Contains(with, "You cannot undo") {
t.Error("the prompt still says the assistant cannot undo")
}
without := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 500, HeightCM: 400, UnitPref: domain.UnitMetric}, "2026-08-23")
if !strings.Contains(without, "no notes") {
t.Error("a garden without notes doesn't say so")
}
}
// TestAClaimedChangeNoToolMadeIsCorrected — the live model, asked to delete a
// journal entry, answered "Done — I've deleted it" having called nothing, and
// the entry was found still there a turn later. The prompt forbids that; the
// run now catches it too: a reply that claims a change, in a turn where no
// tool call changed anything, gets a correction the person can read.
func TestAClaimedChangeNoToolMadeIsCorrected(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
entry, err := svc.CreateJournalEntry(ctx, owner, g.ID, service.JournalInput{Body: "Aphids on the cucumbers."})
if err != nil {
t.Fatalf("journal: %v", err)
}
run := func(steps ...fake.Step) string {
t.Helper()
turn, err := scriptedRunner(t, svc, steps...).Run(ctx, owner, g.ID, "delete that note", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
return turn.Reply
}
corrected := func(reply string) bool { return strings.Contains(reply, "nothing actually changed") }
// No tool at all, a confident claim: corrected.
if r := run(fake.Reply("Done — I've deleted the journal entry about the aphids.")); !corrected(r) {
t.Errorf("a claim with no tool call passed uncorrected: %q", r)
}
// Only reads, then a claim: corrected.
if r := run(toolCall("read_journal", map[string]any{"gardenId": g.ID}), fake.Reply("I've deleted it.")); !corrected(r) {
t.Errorf("a claim over read-only calls passed uncorrected: %q", r)
}
// A tool that FAILED, then a claim: corrected — and the failure names the
// entry and where the ids come from, so a model that reads it has no
// excuse to guess again.
box := NewToolbox(svc, owner, "")
res := box.Execute(ctx, llm.ToolCall{ID: "1", Name: "delete_journal_entry", Arguments: mustJSON(t, map[string]any{"entryId": 999})})
if !res.IsError || !strings.Contains(res.Content, "read_journal") || !strings.Contains(res.Content, "nothing was changed") {
t.Errorf("deleting a missing entry = %q, want a refusal naming read_journal and saying nothing changed", res.Content)
}
if r := run(toolCall("delete_journal_entry", map[string]any{"entryId": 999}), fake.Reply("Done — it's gone.")); !corrected(r) {
t.Errorf("a claim over a failed call passed uncorrected: %q", r)
}
// No claim: nothing appended, whatever the tools did.
if r := run(fake.Reply("That note is still there — want me to delete it?")); corrected(r) {
t.Errorf("an offer was corrected as if it were a claim: %q", r)
}
// Reading the public link is not a change, whatever its tool name.
if r := run(toolCall("public_link", map[string]any{"gardenId": g.ID, "action": "get"}), fake.Reply("Done — I've turned the public link on.")); !corrected(r) {
t.Errorf("a claim over public_link get passed uncorrected: %q", r)
}
// An undo that had nothing to revert is not a change either.
history, _, _ := svc.GardenHistory(ctx, owner, g.ID, 1, 0)
if len(history) > 0 {
if _, _, err := svc.RevertChangeSet(ctx, owner, history[0].ID, domain.SourceUI); err != nil {
t.Fatalf("pre-revert: %v", err)
}
if r := run(toolCall("undo_change", map[string]any{"changeSetId": history[0].ID}), fake.Reply("Undone — it's back the way it was.")); !corrected(r) {
t.Errorf("a claim over an undo that reverted nothing passed uncorrected: %q", r)
}
}
// A real deletion: the claim stands.
r := run(toolCall("delete_journal_entry", map[string]any{"entryId": entry.ID}), fake.Reply("Done — I've deleted the journal entry."))
if corrected(r) {
t.Errorf("a true claim was corrected: %q", r)
}
// A read-only answer that happens to open with a participle is left alone.
if r := run(toolCall("describe_garden", map[string]any{"gardenId": g.ID}), fake.Reply("Updated totals: 0 plantings. Nothing is in the ground.")); corrected(r) {
t.Errorf("an informational reply was corrected: %q", r)
}
if _, _, err := svc.ListJournal(ctx, owner, g.ID, service.JournalQuery{}); err != nil {
t.Fatalf("journal after: %v", err)
}
}