gadfly/cmd/gadfly/wrapup_test.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"testing"

	llm "gitea.stevedudenhoeffer.com/steve/majordomo/llm"
	"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
)

// spinToolCall is a response that asks for the get_diff tool (which succeeds and
// ignores extra args), used to burn agent steps without producing a final
// answer. The args vary by n so successive calls are not byte-identical — that
// dodges the agent's same-call loop guard, exactly as a real reviewer making
// distinct tool calls would.
func spinToolCall(n int) fake.Step {
	return fake.ReplyWith(llm.Response{
		ToolCalls: []llm.ToolCall{{
			ID:        "call",
			Name:      "get_diff",
			Arguments: json.RawMessage(fmt.Sprintf(`{"_n":%d}`, n)),
		}},
		FinishReason: llm.FinishToolCalls,
		Usage:        llm.Usage{InputTokens: 1, OutputTokens: 1},
	})
}

// lastUserText returns the text of the final message in the request, which is
// what a fresh Generate call is reacting to.
func lastUserText(req llm.Request) string {
	if len(req.Messages) == 0 {
		return ""
	}
	return req.Messages[len(req.Messages)-1].Text()
}

// TestReviewExecutor_WrapUpNudgeProducesAnswer: a model that keeps calling tools
// until it is nudged to wrap up should still finish inside its budget — the steer
// message (delivered by the executus wrap-up critic a few steps before the cap)
// arrives and the model writes its answer.
func TestReviewExecutor_WrapUpNudgeProducesAnswer(t *testing.T) {
	t.Setenv("GADFLY_WRAPUP_RESERVE", "4")

	final := "VERDICT: No material issues found."
	nudgeSeen := false
	n := 0
	p := fake.New("fake", fake.WithDefault(func(_ string, req llm.Request) fake.Step {
		if strings.Contains(lastUserText(req), "almost out of your investigation budget") {
			nudgeSeen = true
			return fake.Reply(final)
		}
		n++
		return spinToolCall(n)
	}))
	mdl, err := p.Model("mock")
	if err != nil {
		t.Fatal(err)
	}
	fs, _ := newRepoFS(t.TempDir(), "diff --git a/x b/x\n+y\n")

	rex := newTestReviewExecutor(t, mdl, fs)
	out, err := rex.run(context.Background(), "sys", "task", 12)
	if err != nil {
		t.Fatalf("run should succeed via wrap-up nudge, got error: %v", err)
	}
	if out != final {
		t.Errorf("expected final review %q, got %q", final, out)
	}
	if !nudgeSeen {
		t.Error("the wrap-up nudge was never delivered to the model")
	}
}

// TestReviewExecutor_ExhaustionWithoutAnswerIsError: a model that ignores the
// wrap-up nudge and spins on tools until the step cap produces no final answer.
// The transcript-based forced-finalization fallback was removed in the executus
// re-platform (run.Result does not expose the loop transcript), so the pass now
// surfaces an error — which reviewWithSpecialist renders as an advisory "reviewer
// failed to complete" notice rather than a phantom success.
func TestReviewExecutor_ExhaustionWithoutAnswerIsError(t *testing.T) {
	t.Setenv("GADFLY_WRAPUP_RESERVE", "2")

	n := 0
	p := fake.New("fake", fake.WithDefault(func(_ string, _ llm.Request) fake.Step {
		n++
		return spinToolCall(n) // spin forever, ignoring the wrap-up nudge
	}))
	mdl, err := p.Model("mock")
	if err != nil {
		t.Fatal(err)
	}
	fs, _ := newRepoFS(t.TempDir(), "diff --git a/x b/x\n+y\n")

	rex := newTestReviewExecutor(t, mdl, fs)
	if _, err := rex.run(context.Background(), "sys", "task", 6); err == nil {
		t.Error("run should error when the model exhausts its steps without an answer")
	}
}

func TestWrapUpReserve(t *testing.T) {
	t.Setenv("GADFLY_WRAPUP_RESERVE", "")
	if got := wrapUpReserve(); got != defaultWrapUpReserve {
		t.Errorf("default wrap-up reserve = %d, want %d", got, defaultWrapUpReserve)
	}
	t.Setenv("GADFLY_WRAPUP_RESERVE", "7")
	if got := wrapUpReserve(); got != 7 {
		t.Errorf("wrap-up reserve override = %d, want 7", got)
	}
}