gadfly/cmd/gadfly/lens_concurrency_test.go

package main

import (
	"context"
	"sync"
	"testing"
	"time"

	llm "gitea.stevedudenhoeffer.com/steve/majordomo/llm"
	"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
)

// trackingModel wraps a real llm.Model and records the maximum number of
// concurrent Generate calls, holding each call open briefly so overlap is
// observable. The concurrency tracking + sleep happen OUTSIDE the wrapped
// model's call, so (unlike the fake provider, which holds its mutex while
// invoking its default func) this measures genuine caller-side concurrency.
type trackingModel struct {
	inner   llm.Model
	onEnter func()
	onExit  func()
}

func (m *trackingModel) Generate(ctx context.Context, req llm.Request, opts ...llm.Option) (*llm.Response, error) {
	m.onEnter()
	defer m.onExit()
	time.Sleep(40 * time.Millisecond) // overlap window: concurrent lenses pile up here
	return m.inner.Generate(ctx, req, opts...)
}

func (m *trackingModel) Stream(ctx context.Context, req llm.Request, opts ...llm.Option) (llm.Stream, error) {
	return m.inner.Stream(ctx, req, opts...)
}

func (m *trackingModel) Capabilities() llm.Capabilities { return m.inner.Capabilities() }

// peakTrackingModel returns a model that replies with a clean verdict (so the
// recheck pass is skipped — one Generate per lens) and a getter for the peak
// number of concurrent Generate calls observed.
func peakTrackingModel(t *testing.T) (llm.Model, func() int) {
	t.Helper()
	p := fake.New("fake", fake.WithDefault(func(_ string, _ llm.Request) fake.Step {
		return fake.Reply("No material issues found.")
	}))
	inner, err := p.Model("mock")
	if err != nil {
		t.Fatal(err)
	}

	var mu sync.Mutex
	var inFlight, peak int
	enter := func() {
		mu.Lock()
		defer mu.Unlock()
		inFlight++
		if inFlight > peak {
			peak = inFlight
		}
	}
	exit := func() {
		mu.Lock()
		defer mu.Unlock()
		inFlight--
	}
	return &trackingModel{inner: inner, onEnter: enter, onExit: exit},
		func() int { mu.Lock(); defer mu.Unlock(); return peak }
}

func threeLenses() []Specialist {
	return []Specialist{
		{Name: "security", Title: "🔒 Security", Focus: "x"},
		{Name: "correctness", Title: "🎯 Correctness", Focus: "y"},
		{Name: "error-handling", Title: "🧯 Errors", Focus: "z"},
	}
}

func assertOrderClean(t *testing.T, specs []Specialist, results []specialistResult) {
	t.Helper()
	if len(results) != len(specs) {
		t.Fatalf("got %d results, want %d", len(results), len(specs))
	}
	for i, r := range results {
		if r.spec.Name != specs[i].Name {
			t.Errorf("result %d is %q, want %q (order must follow the specialist list, not finish order)", i, r.spec.Name, specs[i].Name)
		}
		if r.errored {
			t.Errorf("lens %q unexpectedly errored", r.spec.Name)
		}
		if r.verdict != verdictClean {
			t.Errorf("lens %q verdict = %v, want clean", r.spec.Name, r.verdict)
		}
	}
}

// TestRunSpecialists_FansOut: with GADFLY_LENS_CONCURRENCY=3 all three lenses run
// at once (peak == 3), and results still come back in specialist order.
func TestRunSpecialists_FansOut(t *testing.T) {
	t.Setenv("GADFLY_LENS_CONCURRENCY", "3")
	t.Setenv("GADFLY_PROVIDER_LENS_CONCURRENCY", "")
	mdl, peak := peakTrackingModel(t)
	fs, err := newRepoFS(t.TempDir(), "diff --git a/x b/x\n+y\n")
	if err != nil {
		t.Fatal(err)
	}
	specs := threeLenses()

	results := runSpecialists(testEngine(t, mdl, fs), "sys", specs, "task", "diff")

	if got := peak(); got != 3 {
		t.Errorf("peak concurrent lenses = %d, want 3", got)
	}
	assertOrderClean(t, specs, results)
}

// TestRunSpecialists_SequentialByDefault: an unset GADFLY_LENS_CONCURRENCY keeps
// the suite strictly serial (peak == 1) — the historical behavior.
func TestRunSpecialists_SequentialByDefault(t *testing.T) {
	t.Setenv("GADFLY_LENS_CONCURRENCY", "")
	t.Setenv("GADFLY_PROVIDER_LENS_CONCURRENCY", "")
	mdl, peak := peakTrackingModel(t)
	fs, err := newRepoFS(t.TempDir(), "diff --git a/x b/x\n+y\n")
	if err != nil {
		t.Fatal(err)
	}
	specs := threeLenses()

	results := runSpecialists(testEngine(t, mdl, fs), "sys", specs, "task", "diff")

	if got := peak(); got != 1 {
		t.Errorf("peak concurrent lenses = %d, want 1 (sequential by default)", got)
	}
	assertOrderClean(t, specs, results)
}

// TestRunSpecialists_PerProviderFanOut: a per-provider override fans this model's
// lenses out even when the scalar default is serial — the model's lane (from
// GADFLY_MODEL's prefix) is matched against GADFLY_PROVIDER_LENS_CONCURRENCY.
func TestRunSpecialists_PerProviderFanOut(t *testing.T) {
	t.Setenv("GADFLY_MODEL", "m1/qwen3:14b")
	t.Setenv("GADFLY_LENS_CONCURRENCY", "1")             // scalar default: serial
	t.Setenv("GADFLY_PROVIDER_LENS_CONCURRENCY", "m1=3") // but the m1 lane fans out
	mdl, peak := peakTrackingModel(t)
	fs, err := newRepoFS(t.TempDir(), "diff --git a/x b/x\n+y\n")
	if err != nil {
		t.Fatal(err)
	}
	specs := threeLenses()

	results := runSpecialists(testEngine(t, mdl, fs), "sys", specs, "task", "diff")

	if got := peak(); got != 3 {
		t.Errorf("peak concurrent lenses = %d, want 3 (m1 per-provider override)", got)
	}
	assertOrderClean(t, specs, results)
}

// TestLensConcurrency covers the resolution matrix: scalar default, scalar
// override, and per-provider override keyed by the model's resolved lane (same
// lane rule entrypoint.sh uses for GADFLY_PROVIDER_CONCURRENCY).
func TestLensConcurrency(t *testing.T) {
	tests := []struct {
		name     string
		model    string
		provider string
		scalar   string
		provMap  string
		want     int
	}{
		{"default", "", "", "", "", 1},
		{"scalar", "", "", "4", "", 4},
		{"scalar invalid falls back to default", "", "", "nope", "", 1},
		{"bare id uses GADFLY_PROVIDER lane", "qwen3-coder:480b-cloud", "ollama-cloud", "1", "ollama-cloud=3", 3},
		{"bare id with no provider defaults to ollama-cloud lane", "qwen3-coder:480b-cloud", "", "1", "ollama-cloud=7", 7},
		{"prefixed spec uses its prefix lane", "m1/qwen3:14b", "", "1", "m1=2,ollama-cloud=3", 2},
		{"no lane match falls back to scalar", "m5/qwen3.6:35b-mlx", "", "5", "m1=2", 5},
		{"override wins over scalar", "ollama-cloud/x", "", "9", "ollama-cloud=2", 2},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Setenv("GADFLY_MODEL", tt.model)
			t.Setenv("GADFLY_PROVIDER", tt.provider)
			t.Setenv("GADFLY_LENS_CONCURRENCY", tt.scalar)
			t.Setenv("GADFLY_PROVIDER_LENS_CONCURRENCY", tt.provMap)
			if got := lensConcurrency(); got != tt.want {
				t.Errorf("lensConcurrency() = %d, want %d", got, tt.want)
			}
		})
	}
}