executus/run/salvage_test.go

package run_test

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"sync"
	"testing"

	"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
	"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
	"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"

	"gitea.stevedudenhoeffer.com/steve/executus/run"
	"gitea.stevedudenhoeffer.com/steve/executus/tool"
)

// salvageNoopTool is an always-succeeds tool injected via Invocation.ExtraTools
// so a scripted tool-call loop advances (and counts a real tool call) without
// tripping the tool-error guard.
func salvageNoopTool() []llm.Tool {
	return []llm.Tool{{
		Name:        "noop",
		Description: "no-op",
		Handler:     func(context.Context, json.RawMessage) (any, error) { return "ok", nil },
	}}
}

// narratingLoop returns a fake provider whose every reply NARRATES a step and
// calls noop with DISTINCT args (so the same-call-repeat guard never trips), so
// the loop never finalizes and runs until its step / tool-call budget. Each
// step's narration is recoverable by the salvage path.
func narratingLoop() *fake.Provider {
	var mu sync.Mutex
	var n int
	return fake.New("fake", fake.WithDefault(func(_ string, _ llm.Request) fake.Step {
		mu.Lock()
		n++
		step := n
		mu.Unlock()
		return fake.ReplyWith(llm.Response{
			Parts: []llm.Part{llm.Text(fmt.Sprintf("reasoning %d: partial finding %d", step, step))},
			ToolCalls: []llm.ToolCall{{
				ID:        fmt.Sprintf("c%d", step),
				Name:      "noop",
				Arguments: []byte(fmt.Sprintf(`{"n":%d}`, step)),
			}},
		})
	}))
}

// silentLoop is like narratingLoop but writes NO prose — so there is nothing for
// the salvage path to recover.
func silentLoop() *fake.Provider {
	var mu sync.Mutex
	var n int
	return fake.New("fake", fake.WithDefault(func(_ string, _ llm.Request) fake.Step {
		mu.Lock()
		n++
		step := n
		mu.Unlock()
		return fake.ReplyWith(llm.Response{
			ToolCalls: []llm.ToolCall{{
				ID:        fmt.Sprintf("c%d", step),
				Name:      "noop",
				Arguments: []byte(fmt.Sprintf(`{"n":%d}`, step)),
			}},
		})
	}))
}

func modelsFor(m llm.Model) run.ModelResolver {
	return func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil }
}

// TestSalvageSingleLoop_OnRecoversPartialReasoning: with the salvage default ON,
// a single-loop run that exhausts its step budget without a final answer is
// downgraded to a successful PARTIAL result carrying its step narration, instead
// of returning empty output + a hard ErrMaxSteps.
func TestSalvageSingleLoop_OnRecoversPartialReasoning(t *testing.T) {
	fp := narratingLoop()
	m, _ := fp.Model("m")
	ex := run.New(run.Config{
		Registry: tool.NewRegistry(),
		Models:   modelsFor(m),
		Defaults: run.Defaults{SalvageSingleLoopMaxSteps: true},
	})
	res := ex.Run(context.Background(),
		run.RunnableAgent{Name: "researcher", ModelTier: "m", MaxIterations: 3},
		tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
		"do research")

	if res.Err != nil {
		t.Fatalf("salvage ON: budget exhaustion should downgrade to success; got err %v", res.Err)
	}
	if !strings.Contains(res.Output, "partial finding") {
		t.Errorf("salvaged output should contain the step narration; got %q", res.Output)
	}
	if !strings.Contains(res.Output, "step budget") {
		t.Errorf("salvaged output should carry the partial-answer note; got %q", res.Output)
	}
}

// TestSalvageSingleLoop_OffKeepsHardError: with salvage OFF (the zero-value
// default), the same run returns empty output + a hard ErrMaxSteps — the
// structured-output host's clean failure.
func TestSalvageSingleLoop_OffKeepsHardError(t *testing.T) {
	fp := narratingLoop()
	m, _ := fp.Model("m")
	ex := run.New(run.Config{
		Registry: tool.NewRegistry(),
		Models:   modelsFor(m),
		// Defaults zero => SalvageSingleLoopMaxSteps false.
	})
	res := ex.Run(context.Background(),
		run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 3},
		tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
		"go")

	if !errors.Is(res.Err, agent.ErrMaxSteps) {
		t.Fatalf("salvage OFF: expected ErrMaxSteps, got %v", res.Err)
	}
	if strings.TrimSpace(res.Output) != "" {
		t.Errorf("salvage OFF: output should be empty; got %q", res.Output)
	}
}

// TestSalvageSingleLoop_NoProseStillErrors: salvage recovers nothing when the
// loop wrote no prose, so a hard error stands even with salvage ON.
func TestSalvageSingleLoop_NoProseStillErrors(t *testing.T) {
	fp := silentLoop()
	m, _ := fp.Model("m")
	ex := run.New(run.Config{
		Registry: tool.NewRegistry(),
		Models:   modelsFor(m),
		Defaults: run.Defaults{SalvageSingleLoopMaxSteps: true},
	})
	res := ex.Run(context.Background(),
		run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 3},
		tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
		"go")

	if !errors.Is(res.Err, agent.ErrMaxSteps) {
		t.Fatalf("no prose to salvage: ErrMaxSteps should stand, got %v", res.Err)
	}
	if strings.TrimSpace(res.Output) != "" {
		t.Errorf("output should be empty; got %q", res.Output)
	}
}

// TestMaxToolCalls_ForcesExitAndSurfacesSentinel: a run capped at 2 tool calls
// stops after the 2nd (well before its high step ceiling) and surfaces the
// distinct ErrMaxToolCalls sentinel.
func TestMaxToolCalls_ForcesExitAndSurfacesSentinel(t *testing.T) {
	fp := narratingLoop()
	m, _ := fp.Model("m")
	ex := run.New(run.Config{
		Registry: tool.NewRegistry(),
		Models:   modelsFor(m),
		// salvage OFF so the raw sentinel is observable.
	})
	res := ex.Run(context.Background(),
		run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 50, MaxToolCalls: 2},
		tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
		"go")

	if !errors.Is(res.Err, run.ErrMaxToolCalls) {
		t.Fatalf("expected ErrMaxToolCalls, got %v", res.Err)
	}
	// It must NOT read as a plain step-budget exhaustion.
	if !strings.Contains(res.Err.Error(), "max tool calls") {
		t.Errorf("error should name the tool-call cap; got %q", res.Err.Error())
	}
}

// TestMaxToolCalls_SalvagedWhenEnabled: a tool-call-cap exit is treated as budget
// exhaustion, so with salvage ON it recovers the partial reasoning too.
func TestMaxToolCalls_SalvagedWhenEnabled(t *testing.T) {
	fp := narratingLoop()
	m, _ := fp.Model("m")
	ex := run.New(run.Config{
		Registry: tool.NewRegistry(),
		Models:   modelsFor(m),
		Defaults: run.Defaults{SalvageSingleLoopMaxSteps: true},
	})
	res := ex.Run(context.Background(),
		run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 50, MaxToolCalls: 2},
		tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
		"go")

	if res.Err != nil {
		t.Fatalf("cap + salvage: expected downgraded success, got %v", res.Err)
	}
	if !strings.Contains(res.Output, "partial finding") {
		t.Errorf("expected salvaged narration, got %q", res.Output)
	}
}

// TestMaxToolCalls_ZeroIsUnbounded: MaxToolCalls <= 0 imposes no ceiling — a run
// that makes several tool calls then finalizes completes normally (regression:
// unlimited must be a true no-op).
func TestMaxToolCalls_ZeroIsUnbounded(t *testing.T) {
	fp := fake.New("fake")
	fp.Enqueue("m",
		fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c1", Name: "noop", Arguments: []byte(`{"n":1}`)}}}),
		fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c2", Name: "noop", Arguments: []byte(`{"n":2}`)}}}),
		fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c3", Name: "noop", Arguments: []byte(`{"n":3}`)}}}),
		fake.Reply("final answer after 3 tools"),
	)
	m, _ := fp.Model("m")
	ex := run.New(run.Config{
		Registry: tool.NewRegistry(),
		Models:   modelsFor(m),
	})
	res := ex.Run(context.Background(),
		run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 10, MaxToolCalls: 0},
		tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
		"go")

	if res.Err != nil {
		t.Fatalf("unbounded run should complete: %v", res.Err)
	}
	if res.Output != "final answer after 3 tools" {
		t.Errorf("output = %q, want the final answer", res.Output)
	}
}

// TestMaxToolCalls_CapOverridesCriticRaise: the tool-call cap is a HARD budget —
// it stops the run even when a critic is willing to raise the step ceiling far
// past it. Also proves stepCeilingOption composes with a critic.
func TestMaxToolCalls_CapOverridesCriticRaise(t *testing.T) {
	h := &fakeCriticHandle{maxSteps: 50}
	fp := narratingLoop()
	m, _ := fp.Model("m")
	ex := run.New(run.Config{
		Registry: tool.NewRegistry(),
		Models:   modelsFor(m),
		Ports:    run.Ports{Critic: &fakeCritic{h: h}},
	})
	res := ex.Run(context.Background(),
		run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 1, MaxToolCalls: 2, Critic: run.CriticConfig{Enabled: true}},
		tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
		"go")

	if !errors.Is(res.Err, run.ErrMaxToolCalls) {
		t.Fatalf("the tool-call cap must override the critic's raised ceiling; got %v", res.Err)
	}
}