Files
pansy/internal/agent/runtime_test.go
T
steveandClaude Fable 5 d0ca56b79b
Build image / build-and-push (push) Successful in 23s
Gadfly review (reusable) / review (pull_request) Successful in 7m19s
Adversarial Review (Gadfly) / review (pull_request) Successful in 7m20s
Agent: a turn that changed nothing cannot say it did
Live, asked to delete a journal entry and later a seed lot, the model
answered "Done — I've deleted it" both times having deleted nothing; each
was still there a turn later. The prompt already forbade that. Now the run
catches it: honestReply appends a correction when the reply claims a change
("Done", "I've deleted…") and no non-read-only tool call succeeded, and logs
the steps so the mechanism can be read off the log next time.

Alongside: the id-taking tools turn a bare "not found" into a message that
names what was missing and which tool lists the ids ("nothing was changed"),
the two delete descriptions say to look the id up in THIS turn, and the
prompt says an error result means the thing did not happen.

Co-Authored-By: Claude Fable 5 <[email protected]>
2026-08-23 02:37:06 -04:00

654 lines
26 KiB
Go

package agent
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"unicode/utf8"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
"gitea.stevedudenhoeffer.com/steve/pansy/internal/domain"
"gitea.stevedudenhoeffer.com/steve/pansy/internal/service"
)
// scriptedRunner wires a Runner to a fake provider so the whole loop — tools,
// change-set scoping, loop guards — is exercised with no live model.
func scriptedRunner(t *testing.T, svc *service.Service, steps ...fake.Step) *Runner {
t.Helper()
p := fake.New("fake")
for _, s := range steps {
p.Enqueue("m", s)
}
model, err := p.Model("m")
if err != nil {
t.Fatalf("fake model: %v", err)
}
return &Runner{svc: svc, model: model}
}
// toolCall scripts one model turn that calls a tool.
func toolCall(name string, args any) fake.Step {
raw, _ := json.Marshal(args)
return fake.ReplyWith(llm.Response{
FinishReason: llm.FinishToolCalls,
ToolCalls: []llm.ToolCall{{ID: "c1", Name: name, Arguments: raw}},
})
}
// TestTurnIsOneChangeSet is the acceptance criterion the whole "act freely"
// posture rests on: a turn that clears a bed and replants it — one object edit
// and many planting inserts — has to undo as ONE action, not thirteen.
func TestTurnIsOneChangeSet(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
garlic := mustPlant(t, svc, owner, "Garlic", 15, "🧄")
cucumber := mustPlant(t, svc, owner, "Cucumber", 45, "🥒")
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{
Kind: domain.KindBed, Name: "Garlic bed", XCM: 1000, YCM: 1000, WidthCM: 400, HeightCM: 400,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
if _, err := svc.FillNamedRegion(ctx, owner, bed.ID, "all", garlic.ID, nil, service.FillClump, nil); err != nil {
t.Fatalf("seed garlic: %v", err)
}
before, _, err := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if err != nil {
t.Fatalf("history: %v", err)
}
r := scriptedRunner(t, svc,
toolCall("clear_object", map[string]any{"objectId": bed.ID}),
toolCall("fill_region", map[string]any{"objectId": bed.ID, "region": "all", "plantId": cucumber.ID}),
fake.Reply("Cleared the garlic and replanted the bed with cucumbers."),
)
turn, err := r.Run(ctx, owner, g.ID, "change the garlic bed to cucumbers this year", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
if turn.ChangeSetID == nil {
t.Fatal("a turn that changed things produced no change set")
}
if !strings.Contains(turn.Reply, "cucumbers") {
t.Errorf("reply = %q", turn.Reply)
}
// Exactly ONE new change set, whatever the model did inside the turn.
after, _, err := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if err != nil {
t.Fatalf("history: %v", err)
}
if len(after)-len(before) != 1 {
t.Fatalf("turn produced %d change sets, want exactly 1", len(after)-len(before))
}
cs := after[0]
if cs.Source != domain.SourceAgent {
t.Errorf("source = %q, want agent", cs.Source)
}
if cs.Summary != "change the garlic bed to cucumbers this year" {
t.Errorf("summary = %q, want the user's own words", cs.Summary)
}
// The bed really is cucumbers now.
full, _ := svc.GardenFull(ctx, owner, g.ID, nil)
if len(full.Plantings) == 0 {
t.Fatal("bed ended up empty")
}
for _, p := range full.Plantings {
if p.PlantID != cucumber.ID {
t.Errorf("unexpected plant %d still in the bed", p.PlantID)
}
}
// And one undo puts it back.
if _, conflicts, err := svc.RevertChangeSet(ctx, owner, *turn.ChangeSetID, domain.SourceUI); err != nil || len(conflicts) != 0 {
t.Fatalf("undo: err=%v conflicts=%+v", err, conflicts)
}
full, _ = svc.GardenFull(ctx, owner, g.ID, nil)
for _, p := range full.Plantings {
if p.PlantID != garlic.ID {
t.Errorf("after undo the bed holds plant %d, want the garlic back", p.PlantID)
}
}
}
// TestViewerGetsAnExplainableRefusal — the ACL story only works if the model can
// narrate the refusal, so a tool denial has to reach it as a tool RESULT it can
// read, not as a 500 that ends the run.
func TestViewerGetsAnExplainableRefusal(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
viewer, err := svc.Register(ctx, service.RegisterInput{Email: "[email protected]", DisplayName: "V", Password: "password123"})
if err != nil {
t.Fatalf("register: %v", err)
}
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{
Kind: domain.KindBed, XCM: 1000, YCM: 1000, WidthCM: 400, HeightCM: 400,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
if _, err := svc.AddShare(ctx, owner, g.ID, "[email protected]", domain.RoleViewer); err != nil {
t.Fatalf("share: %v", err)
}
// A viewer can't open a change set at all, so the turn is refused up front —
// before any model call — and the API turns that into a plain explanation.
r := scriptedRunner(t, svc, fake.Reply("unused"))
_, err = r.Run(ctx, viewer.ID, g.ID, "plant garlic in that bed", "", nil, nil)
if !errors.Is(err, domain.ErrForbidden) {
t.Fatalf("viewer turn err = %v, want ErrForbidden", err)
}
// And at the tool layer, a refusal comes back as a readable tool result
// rather than killing the run.
box := NewToolbox(svc, viewer.ID, "")
raw, _ := json.Marshal(map[string]any{"objectId": bed.ID})
res := box.Execute(ctx, llm.ToolCall{ID: "1", Name: "clear_object", Arguments: raw})
if !res.IsError {
t.Fatal("a viewer's clear_object succeeded")
}
if res.Content == "" {
t.Error("the refusal carried no text for the model to explain")
}
}
// TestRunStopsAtTheStepCap — a model that gets wedged should stop on its own,
// and what it managed to do must still be recorded and undoable.
func TestRunStopsAtTheStepCap(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
// More describe_garden calls than the cap allows, forever.
steps := make([]fake.Step, 0, maxSteps+4)
for i := 0; i < maxSteps+4; i++ {
steps = append(steps, toolCall("describe_garden", map[string]any{"gardenId": g.ID}))
}
r := scriptedRunner(t, svc, steps...)
turn, err := r.Run(ctx, owner, g.ID, "look at the garden", "", nil, nil)
if err != nil {
t.Fatalf("a capped run should end cleanly, got %v", err)
}
if !turn.Truncated {
t.Error("turn.Truncated = false; the run hit the cap and should say so")
}
if turn.Reply == "" {
t.Error("a capped run said nothing; silence reads as a hang")
}
// It only read, so there is nothing to undo — and no empty change set either.
if turn.ChangeSetID != nil {
t.Errorf("a read-only turn produced change set %d", *turn.ChangeSetID)
}
}
// TestReadOnlyTurnWritesNoChangeSet — asking a question must not litter the
// history with empty entries.
func TestReadOnlyTurnWritesNoChangeSet(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
before, _, _ := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
r := scriptedRunner(t, svc,
toolCall("describe_garden", map[string]any{"gardenId": g.ID}),
fake.Reply("It's empty — nothing planted yet."),
)
turn, err := r.Run(ctx, owner, g.ID, "what's in the garden?", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
if turn.ChangeSetID != nil {
t.Errorf("a question produced change set %d", *turn.ChangeSetID)
}
after, _, _ := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if len(after) != len(before) {
t.Errorf("history grew by %d for a read-only turn", len(after)-len(before))
}
}
// TestNewRunnerNeedsConfiguration — an instance with no key or no model must not
// get a half-built runner; the caller treats the error as "no assistant" and
// carries on. (Whether the assistant is ENABLED is resolved before NewRunner is
// reached, so it isn't NewRunner's concern any more.)
func TestNewRunnerNeedsConfiguration(t *testing.T) {
svc, _ := newAgentTestService(t)
for _, tc := range []struct{ key, model string }{
{"", "ollama-cloud/x"},
{"k", ""},
{"k", " "},
} {
if _, err := NewRunner(svc, tc.key, tc.model); err == nil {
t.Errorf("NewRunner accepted key=%q model=%q", tc.key, tc.model)
}
}
// A model spec naming a provider that doesn't exist is a configuration
// error, not a panic at first use.
if _, err := NewRunner(svc, "k", "nonesuch/model"); err == nil {
t.Error("NewRunner accepted an unresolvable model spec")
}
}
// TestTurnSummaryFitsAHistoryRow — the user's own words are the most useful
// label for a change set, but a paragraph would wreck the list.
func TestTurnSummaryFitsAHistoryRow(t *testing.T) {
if got := turnSummary(" change the garlic bed\n to cucumbers "); got != "change the garlic bed to cucumbers" {
t.Errorf("turnSummary = %q", got)
}
long := turnSummary(strings.Repeat("plant garlic ", 40))
if len(long) > 130 || !strings.HasSuffix(long, "…") {
t.Errorf("long summary = %q (%d chars)", long, len(long))
}
}
// TestSystemPromptStatesTheCompassConvention — -y being north is not guessable,
// and a model that assumes otherwise plants the wrong end of the bed.
func TestSystemPromptStatesTheCompassConvention(t *testing.T) {
p := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 500, HeightCM: 400, UnitPref: domain.UnitImperial}, "2026-08-22")
for _, want := range []string{"NORTH", "-y", "CENTIMETERS", "Plot", "version"} {
if !strings.Contains(p, want) {
t.Errorf("system prompt is missing %q:\n%s", want, p)
}
}
if !strings.Contains(p, fmt.Sprintf("%.0f", 500.0)) {
t.Error("system prompt doesn't state the garden's size")
}
}
// TestPartialWorkSurvivesATimeout is the finding that mattered most on this PR.
//
// The run context carries a timeout. When it fires, WithChangeSet's recovery
// path has to record what already committed — and doing that with the SAME
// dead context would fail, losing the history for changes that really happened.
// The user-facing message says "anything I'd already changed is in History", so
// this isn't just a gap, it's a promise the code has to keep.
func TestPartialWorkSurvivesATimeout(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{
Kind: domain.KindBed, Name: "Bed", XCM: 1000, YCM: 1000, WidthCM: 400, HeightCM: 400,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
before, _, _ := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
// A turn that renames the bed, then dies with the context already cancelled.
cancelled, cancel := context.WithCancel(ctx)
r := scriptedRunner(t, svc,
toolCall("move_object", map[string]any{
"objectId": bed.ID, "xCm": 600.0, "yCm": 600.0, "version": bed.Version,
}),
fake.Step{Err: context.DeadlineExceeded},
)
// Cancel once the first tool call has landed, so the failure path runs with a
// dead context — exactly the timeout case.
go func() {
time.Sleep(50 * time.Millisecond)
cancel()
}()
_, err = r.Run(cancelled, owner, g.ID, "move the bed", "", nil, nil)
if err == nil {
t.Fatal("expected the turn to fail")
}
// The move committed, so it must be in history and undoable.
after, _, herr := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if herr != nil {
t.Fatalf("history: %v", herr)
}
if len(after) != len(before)+1 {
t.Fatalf("the failed turn recorded %d change sets, want 1 — its work is otherwise un-undoable",
len(after)-len(before))
}
if !strings.Contains(after[0].Summary, "failed partway") {
t.Errorf("summary = %q, want it marked as partial", after[0].Summary)
}
if _, conflicts, rerr := svc.RevertChangeSet(ctx, owner, after[0].ID, domain.SourceUI); rerr != nil || len(conflicts) != 0 {
t.Fatalf("the partial turn should be undoable: err=%v conflicts=%+v", rerr, conflicts)
}
o, _ := svc.DescribeGarden(ctx, owner, g.ID, nil)
if len(o.Objects) > 0 && o.Objects[0].XCM != bed.XCM {
t.Errorf("undo left the bed at %v, want %v", o.Objects[0].XCM, bed.XCM)
}
}
// TestTurnSummaryTrimsByRunes — slicing a byte offset would cut a multibyte
// character in half and store invalid UTF-8 in the history summary.
func TestTurnSummaryTrimsByRunes(t *testing.T) {
// 200 multibyte runes: a byte slice at 120 would land mid-character.
got := turnSummary(strings.Repeat("🌱", 200))
if !utf8.ValidString(got) {
t.Errorf("turnSummary produced invalid UTF-8: %q", got)
}
if !strings.HasSuffix(got, "…") {
t.Errorf("long summary should be elided, got %q", got)
}
if n := utf8.RuneCountInString(got); n > 121 {
t.Errorf("summary is %d runes, want it trimmed", n)
}
}
// TestSystemPromptKnowsTheDayAndTheGardenersUnits — two things the live model
// got wrong for want of being told: it dated journal entries with the year it
// remembered from training, and answered a feet-and-inches gardener in
// centimeters. The conduct rules are checked by their load-bearing phrases.
func TestSystemPromptKnowsTheDayAndTheGardenersUnits(t *testing.T) {
imperial := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 731.52, HeightCM: 731.52, UnitPref: domain.UnitImperial}, "2026-08-22")
for _, want := range []string{
"Today is 2026-08-22",
"feet and inches",
"24.0 x 24.0 ft",
"never describe a change you did not make",
"undo_change",
"Undo this",
"rather than clearing beds on a guess",
"not to narrate your own planting",
`"Plot — <year>"`,
"remove_plantings",
"move_planting",
"list_plantings",
} {
if !strings.Contains(imperial, want) {
t.Errorf("imperial prompt is missing %q", want)
}
}
metric := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 500, HeightCM: 400, UnitPref: domain.UnitMetric}, "2026-08-22")
if strings.Contains(metric, "feet and inches") {
t.Error("metric prompt tells the model to answer in feet and inches")
}
if !strings.Contains(metric, "500 x 400 cm") {
t.Error("metric prompt doesn't state the garden's size in cm")
}
}
// TestRunRejectsAMalformedToday — the date reaches every tool as a default, so a
// bad one must stop the turn before the model runs, not fail its first fill.
func TestRunRejectsAMalformedToday(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
r := scriptedRunner(t, svc, fake.Reply("unused"))
if _, err := r.Run(ctx, owner, g.ID, "hello", "yesterday", nil, nil); !errors.Is(err, domain.ErrInvalidInput) {
t.Errorf("Run with today=%q: err = %v, want ErrInvalidInput", "yesterday", err)
}
}
// TestTurnDatesItsWorkTheGardenersDay — what a turn plants is dated the day the
// gardener sent it, not the server's UTC day (which is tomorrow by nine in the
// evening in Ohio) and not a day the model chose.
func TestTurnDatesItsWorkTheGardenersDay(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
garlic := mustPlant(t, svc, owner, "Garlic", 15, "🧄")
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{
Kind: domain.KindBed, Name: "Bed", XCM: 1000, YCM: 1000, WidthCM: 200, HeightCM: 200,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
r := scriptedRunner(t, svc,
toolCall("fill_region", map[string]any{"objectId": bed.ID, "region": "all", "plantId": garlic.ID}),
fake.Reply("Filled the bed with garlic."),
)
if _, err := r.Run(ctx, owner, g.ID, "fill the bed with garlic", "2026-08-22", nil, nil); err != nil {
t.Fatalf("Run: %v", err)
}
full, err := svc.GardenFull(ctx, owner, g.ID, nil)
if err != nil {
t.Fatalf("GardenFull: %v", err)
}
if len(full.Plantings) == 0 {
t.Fatal("the turn planted nothing")
}
for _, p := range full.Plantings {
if p.PlantedAt == nil || *p.PlantedAt != "2026-08-22" {
t.Errorf("plop %d plantedAt = %v, want the gardener's day 2026-08-22", p.ID, p.PlantedAt)
}
}
}
// TestTurnOnAnotherGardenFilesHistoryThere — a turn is scoped to one garden, but
// nothing stops the model from pointing a tool at an object in another garden
// the person can edit ("do the same in my other garden"). Those revisions must
// land in THAT garden's history, as the agent's work, where its undo can see
// them — not in the open scope, where undoing this turn would quietly revert
// rows in a garden the person isn't looking at.
func TestTurnOnAnotherGardenFilesHistoryThere(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
a, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "A", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden A: %v", err)
}
b, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "B", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden B: %v", err)
}
bedB, err := svc.CreateObject(ctx, owner, b.ID, service.ObjectInput{
Kind: domain.KindBed, Name: "Bed", XCM: 1000, YCM: 1000, WidthCM: 200, HeightCM: 200,
})
if err != nil {
t.Fatalf("bed: %v", err)
}
beforeA, _, _ := svc.GardenHistory(ctx, owner, a.ID, 0, 0)
beforeB, _, _ := svc.GardenHistory(ctx, owner, b.ID, 0, 0)
r := scriptedRunner(t, svc,
toolCall("update_object", map[string]any{"objectId": bedB.ID, "version": bedB.Version, "name": "Renamed from A"}),
fake.Reply("Renamed the bed in B."),
)
turn, err := r.Run(ctx, owner, a.ID, "rename the bed in my other garden", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
if turn.ChangeSetID != nil {
t.Errorf("the turn on A produced change set %d, but it changed nothing in A", *turn.ChangeSetID)
}
afterA, _, _ := svc.GardenHistory(ctx, owner, a.ID, 0, 0)
if len(afterA) != len(beforeA) {
t.Errorf("A's history grew by %d for a change made in B", len(afterA)-len(beforeA))
}
afterB, _, _ := svc.GardenHistory(ctx, owner, b.ID, 0, 0)
if len(afterB) != len(beforeB)+1 {
t.Fatalf("B's history grew by %d, want 1", len(afterB)-len(beforeB))
}
if cs := afterB[0]; cs.Source != domain.SourceAgent || cs.AgentRunID == nil {
t.Errorf("B's entry = source %q, run %v; want the agent's, with its run id", cs.Source, cs.AgentRunID)
}
// And it undoes from B, where the person would look for it.
if _, conflicts, err := svc.RevertChangeSet(ctx, owner, afterB[0].ID, domain.SourceUI); err != nil || len(conflicts) != 0 {
t.Fatalf("undo from B: err=%v conflicts=%+v", err, conflicts)
}
d, _ := svc.DescribeGarden(ctx, owner, b.ID, nil)
if len(d.Objects) != 1 || d.Objects[0].Name != "Bed" {
t.Errorf("after undo B's bed is %+v, want its original name back", d.Objects)
}
}
// TestTurnThatOnlyUndoesIsItselfUndoable — a revert is its own change set,
// outside the turn's scope, so a turn that did nothing but undo would come back
// with no change of its own; the reply would then be the one change in the
// conversation without an "Undo this". It gets the revert instead — a redo.
func TestTurnThatOnlyUndoesIsItselfUndoable(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
beets := mustPlant(t, svc, owner, "Beets", 10, "🫜")
bed, err := svc.CreateObject(ctx, owner, g.ID, service.ObjectInput{Kind: domain.KindBed, Name: "Bed", XCM: 1000, YCM: 1000, WidthCM: 400, HeightCM: 400})
if err != nil {
t.Fatalf("bed: %v", err)
}
// The beets went in by hand in the editor: the change the person wants undone.
if _, err := svc.FillNamedRegion(ctx, owner, bed.ID, "all", beets.ID, nil, service.FillClump, nil); err != nil {
t.Fatalf("plant beets: %v", err)
}
history, _, err := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if err != nil || len(history) == 0 {
t.Fatalf("history: %v (%d entries)", err, len(history))
}
planted := history[0]
r := scriptedRunner(t, svc,
toolCall("read_history", map[string]any{"gardenId": g.ID}),
toolCall("undo_change", map[string]any{"changeSetId": planted.ID}),
fake.Reply("Undone — the beets are out of the bed again."),
)
turn, err := r.Run(ctx, owner, g.ID, "undo the beets", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
full, err := svc.GardenFull(ctx, owner, g.ID, nil)
if err != nil {
t.Fatalf("GardenFull: %v", err)
}
if len(full.Plantings) != 0 {
t.Fatalf("%d beets still in the bed after the undo", len(full.Plantings))
}
after, _, _ := svc.GardenHistory(ctx, owner, g.ID, 0, 0)
if len(after) != len(history)+1 {
t.Fatalf("history grew by %d, want exactly the revert", len(after)-len(history))
}
revert := after[0]
if revert.Source != domain.SourceAgent || revert.RevertsID == nil || *revert.RevertsID != planted.ID {
t.Errorf("newest entry = %+v; want the agent's revert of %d", revert, planted.ID)
}
if turn.ChangeSetID == nil || *turn.ChangeSetID != revert.ID {
t.Fatalf("turn.ChangeSetID = %v, want the revert %d so the reply can offer a redo", turn.ChangeSetID, revert.ID)
}
// And "Undo this" on that reply is a redo.
if _, conflicts, err := svc.RevertChangeSet(ctx, owner, *turn.ChangeSetID, domain.SourceUI); err != nil || len(conflicts) != 0 {
t.Fatalf("redo: err=%v conflicts=%+v", err, conflicts)
}
full, _ = svc.GardenFull(ctx, owner, g.ID, nil)
if len(full.Plantings) == 0 {
t.Error("redoing the turn did not put the beets back")
}
}
// TestSystemPromptCarriesTheGardenersNotes — the notes are the assistant's
// memory: what the gardener told it about the place comes back on every turn,
// quoted as their words rather than pasted as instructions.
func TestSystemPromptCarriesTheGardenersNotes(t *testing.T) {
with := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 500, HeightCM: 400, UnitPref: domain.UnitMetric,
Notes: "Zone 6a.\nLast frost \"usually\" May 10."}, "2026-08-23")
for _, want := range []string{
`"Zone 6a.\nLast frost \"usually\" May 10."`,
"update_garden",
"undo_change",
"describe_garden with a year",
"readyAround",
"create_garden",
"confirmed=true",
} {
if !strings.Contains(with, want) {
t.Errorf("prompt is missing %q", want)
}
}
if strings.Contains(with, "You cannot undo") {
t.Error("the prompt still says the assistant cannot undo")
}
without := systemPrompt(&domain.Garden{ID: 1, Name: "Plot", WidthCM: 500, HeightCM: 400, UnitPref: domain.UnitMetric}, "2026-08-23")
if !strings.Contains(without, "no notes") {
t.Error("a garden without notes doesn't say so")
}
}
// TestAClaimedChangeNoToolMadeIsCorrected — the live model, asked to delete a
// journal entry, answered "Done — I've deleted it" having called nothing, and
// the entry was found still there a turn later. The prompt forbids that; the
// run now catches it too: a reply that claims a change, in a turn where no
// tool call changed anything, gets a correction the person can read.
func TestAClaimedChangeNoToolMadeIsCorrected(t *testing.T) {
ctx := context.Background()
svc, owner := newAgentTestService(t)
g, err := svc.CreateGarden(ctx, owner, service.GardenInput{Name: "Plot", WidthCM: 2000, HeightCM: 2000})
if err != nil {
t.Fatalf("garden: %v", err)
}
entry, err := svc.CreateJournalEntry(ctx, owner, g.ID, service.JournalInput{Body: "Aphids on the cucumbers."})
if err != nil {
t.Fatalf("journal: %v", err)
}
run := func(steps ...fake.Step) string {
t.Helper()
turn, err := scriptedRunner(t, svc, steps...).Run(ctx, owner, g.ID, "delete that note", "", nil, nil)
if err != nil {
t.Fatalf("Run: %v", err)
}
return turn.Reply
}
corrected := func(reply string) bool { return strings.Contains(reply, "nothing actually changed") }
// No tool at all, a confident claim: corrected.
if r := run(fake.Reply("Done — I've deleted the journal entry about the aphids.")); !corrected(r) {
t.Errorf("a claim with no tool call passed uncorrected: %q", r)
}
// Only reads, then a claim: corrected.
if r := run(toolCall("read_journal", map[string]any{"gardenId": g.ID}), fake.Reply("I've deleted it.")); !corrected(r) {
t.Errorf("a claim over read-only calls passed uncorrected: %q", r)
}
// A tool that FAILED, then a claim: corrected — and the failure names the
// entry and where the ids come from, so a model that reads it has no
// excuse to guess again.
box := NewToolbox(svc, owner, "")
res := box.Execute(ctx, llm.ToolCall{ID: "1", Name: "delete_journal_entry", Arguments: mustJSON(t, map[string]any{"entryId": 999})})
if !res.IsError || !strings.Contains(res.Content, "read_journal") || !strings.Contains(res.Content, "nothing was changed") {
t.Errorf("deleting a missing entry = %q, want a refusal naming read_journal and saying nothing changed", res.Content)
}
if r := run(toolCall("delete_journal_entry", map[string]any{"entryId": 999}), fake.Reply("Done — it's gone.")); !corrected(r) {
t.Errorf("a claim over a failed call passed uncorrected: %q", r)
}
// No claim: nothing appended, whatever the tools did.
if r := run(fake.Reply("That note is still there — want me to delete it?")); corrected(r) {
t.Errorf("an offer was corrected as if it were a claim: %q", r)
}
// A real deletion: the claim stands.
r := run(toolCall("delete_journal_entry", map[string]any{"entryId": entry.ID}), fake.Reply("Done — I've deleted the journal entry."))
if corrected(r) {
t.Errorf("a true claim was corrected: %q", r)
}
if _, _, err := svc.ListJournal(ctx, owner, g.ID, service.JournalQuery{}); err != nil {
t.Fatalf("journal after: %v", err)
}
}