feat: cross-model consensus consolidation (one ranked comment, not N walls) (#17)

Co-authored-by: Steve Dudenhoeffer <[email protected]>
Co-committed-by: Steve Dudenhoeffer <[email protected]>

cmd/gadfly/consensus_test.go

@@ -0,0 +1,217 @@
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestClusterFindingsAgreementAndTolerance(t *testing.T) {
+	models := []modelFindings{
+		{Model: "m1", Verdict: "Blocking issues found", Findings: []outFinding{
+			{Lens: "security", File: "a.go", Line: 10, Severity: "high", Title: "auth bypass"},
+			{Lens: "perf", File: "b.go", Line: 5, Severity: "trivial", Title: "tiny nit"},
+		}},
+		{Model: "m2", Verdict: "Minor issues", Findings: []outFinding{
+			{Lens: "security", File: "a.go", Line: 11, Severity: "critical", Title: "auth bypass (crit)"}, // within tolerance of a.go:10
+		}},
+		{Model: "m3", Verdict: "Minor issues", Findings: []outFinding{
+			{Lens: "correctness", File: "a.go", Line: 10, Severity: "medium", Title: "auth bypass"},
+		}},
+	}
+	clusters := clusterFindings(models)
+	if len(clusters) != 2 {
+		t.Fatalf("want 2 clusters (a.go:10±, b.go:5), got %d: %+v", len(clusters), clusters)
+	}
+	// First cluster (highest agreement) is the a.go auth one: 3 models, severity
+	// escalated to critical, representative line the smallest (10).
+	c := clusters[0]
+	if len(c.models) != 3 {
+		t.Errorf("want 3 models on the top cluster, got %d", len(c.models))
+	}
+	if c.severity != "critical" {
+		t.Errorf("want escalated severity critical, got %q", c.severity)
+	}
+	if c.line != 10 {
+		t.Errorf("want representative line 10, got %d", c.line)
+	}
+	if !c.lenses["security"] || !c.lenses["correctness"] {
+		t.Errorf("want union of lenses, got %v", c.lenses)
+	}
+}
+
+func TestRenderConsensusFoldsSingleModelNits(t *testing.T) {
+	models := []modelFindings{
+		{Model: "m1", Provider: "p", Verdict: "Blocking issues found", Markdown: "m1 detail", Findings: []outFinding{
+			{Lens: "security", File: "a.go", Line: 10, Severity: "high", Title: "auth bypass"},
+			{Lens: "perf", File: "b.go", Line: 5, Severity: "trivial", Title: "tiny nit"},
+		}},
+		{Model: "m2", Provider: "p", Verdict: "Minor issues", Markdown: "m2 detail", Findings: []outFinding{
+			{Lens: "security", File: "a.go", Line: 10, Severity: "high", Title: "auth bypass"},
+		}},
+	}
+	out := renderConsensus(models)
+	// Headline table: the agreed finding with a 2/2 badge.
+	if !strings.Contains(out, "2/2") {
+		t.Errorf("expected a 2/2 agreement badge in headline:\n%s", out)
+	}
+	if !strings.Contains(out, "auth bypass") || !strings.Contains(out, "a.go:10") {
+		t.Errorf("headline missing the consensus finding:\n%s", out)
+	}
+	// The lone trivial finding is folded, not in the headline table.
+	if !strings.Contains(out, "single-model finding") {
+		t.Errorf("expected a folded single-model section:\n%s", out)
+	}
+	// Per-model detail is preserved (folded).
+	if !strings.Contains(out, "m1 detail") || !strings.Contains(out, "m2 detail") {
+		t.Errorf("per-model detail not preserved:\n%s", out)
+	}
+}
+
+func TestRenderConsensusHighSeverityLoneFindingStaysHeadline(t *testing.T) {
+	// A single model, single critical finding must still surface in the headline
+	// (not be folded as "low confidence").
+	models := []modelFindings{
+		{Model: "solo", Verdict: "Blocking issues found", Markdown: "x", Findings: []outFinding{
+			{Lens: "security", File: "a.go", Line: 1, Severity: "critical", Title: "rce"},
+		}},
+	}
+	out := renderConsensus(models)
+	headline := out
+	if i := strings.Index(out, "single-model finding"); i >= 0 {
+		headline = out[:i]
+	}
+	if !strings.Contains(headline, "rce") {
+		t.Errorf("lone critical should be in the headline, not folded:\n%s", out)
+	}
+}
+
+func TestClusterSlidingWindowMergesChain(t *testing.T) {
+	// Findings at 10, 13, 16 (each 3 apart) from three models must merge into ONE
+	// cluster — the window slides with the span instead of anchoring at line 10.
+	models := []modelFindings{
+		{Model: "m1", Findings: []outFinding{{Lens: "x", File: "a.go", Line: 10, Severity: "medium", Title: "t"}}},
+		{Model: "m2", Findings: []outFinding{{Lens: "x", File: "a.go", Line: 13, Severity: "medium", Title: "t"}}},
+		{Model: "m3", Findings: []outFinding{{Lens: "x", File: "a.go", Line: 16, Severity: "medium", Title: "t"}}},
+	}
+	clusters := clusterFindings(models)
+	if len(clusters) != 1 {
+		t.Fatalf("chain 10/13/16 should merge into 1 cluster, got %d", len(clusters))
+	}
+	if len(clusters[0].models) != 3 {
+		t.Errorf("want 3 models in the merged cluster, got %d", len(clusters[0].models))
+	}
+}
+
+func TestRenderConsensusExcludesErroredFromDenominator(t *testing.T) {
+	models := []modelFindings{
+		{Model: "m1", Verdict: "Minor issues", Markdown: "a", Findings: []outFinding{
+			{Lens: "security", File: "a.go", Line: 9, Severity: "medium", Title: "leak"}}},
+		{Model: "m2", Verdict: "Minor issues", Markdown: "b", Findings: []outFinding{
+			{Lens: "security", File: "a.go", Line: 9, Severity: "medium", Title: "leak"}}},
+		{Model: "broken", Verdict: "reviewer failed", Errored: true, Markdown: "boom"},
+	}
+	out := renderConsensus(models)
+	// Denominator is the 2 effective models, not 3; the failure is noted.
+	if !strings.Contains(out, "2/2") {
+		t.Errorf("errored model must be excluded from the denominator (want 2/2):\n%s", out)
+	}
+	if !strings.Contains(out, "1 failed") {
+		t.Errorf("expected a '1 failed' note:\n%s", out)
+	}
+	if !strings.Contains(out, "reviewer failed") {
+		t.Errorf("errored model should still appear (folded) as failed:\n%s", out)
+	}
+}
+
+func TestRenderConsensusLoneHighFolds(t *testing.T) {
+	// A single-model HIGH (not critical) folds — only consensus or a lone CRITICAL
+	// earns the headline, so a lone Blocking-lens finding doesn't reintroduce noise.
+	models := []modelFindings{
+		{Model: "solo", Verdict: "Blocking issues found", Markdown: "x", Findings: []outFinding{
+			{Lens: "security", File: "a.go", Line: 1, Severity: "high", Title: "maybe-bug"}}},
+	}
+	out := renderConsensus(models)
+	head := out
+	if i := strings.Index(out, "single-model finding"); i >= 0 {
+		head = out[:i]
+	}
+	if strings.Contains(head, "maybe-bug") {
+		t.Errorf("a lone HIGH should fold, not headline:\n%s", out)
+	}
+}
+
+func TestWriteAndConsolidateRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+
+	// Two model artifacts on disk.
+	write := func(name string, mf modelFindings) {
+		data, _ := json.Marshal(mf)
+		if err := os.WriteFile(filepath.Join(dir, name), data, 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("m1.json", modelFindings{Model: "m1", Provider: "ollama", Verdict: "Minor issues", Markdown: "md1",
+		Findings: []outFinding{{Lens: "security", File: "x.go", Line: 3, Severity: "medium", Title: "leak"}}})
+	write("m2.json", modelFindings{Model: "m2", Provider: "ollama", Verdict: "Minor issues", Markdown: "md2",
+		Findings: []outFinding{{Lens: "security", File: "x.go", Line: 3, Severity: "high", Title: "leak"}}})
+	// A junk file must be skipped, not crash consolidation.
+	if err := os.WriteFile(filepath.Join(dir, "notes.txt"), []byte("ignore me"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Setenv("GADFLY_CONSOLIDATE_DIR", dir)
+	// runConsolidate prints to stdout; capture it.
+	out := captureStdout(t, func() {
+		if err := runConsolidate(); err != nil {
+			t.Fatalf("runConsolidate: %v", err)
+		}
+	})
+	if !strings.HasPrefix(strings.TrimSpace(out), consensusMarker) {
+		t.Errorf("consolidated output must lead with the marker:\n%s", out)
+	}
+	if !strings.Contains(out, "2/2") || !strings.Contains(out, "x.go:3") {
+		t.Errorf("expected the agreed x.go:3 finding at 2/2:\n%s", out)
+	}
+}
+
+func TestRunConsolidateEmptyDirErrors(t *testing.T) {
+	t.Setenv("GADFLY_CONSOLIDATE_DIR", t.TempDir())
+	if err := runConsolidate(); err == nil {
+		t.Error("want an error for an empty consolidate dir (entrypoint falls back)")
+	}
+}
+
+// captureStdout redirects os.Stdout for the duration of fn and returns what was
+// written.
+func captureStdout(t *testing.T, fn func()) string {
+	t.Helper()
+	orig := os.Stdout
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	os.Stdout = w
+	defer func() { os.Stdout = orig }() // restore even if fn panics
+	done := make(chan string)
+	go func() {
+		var sb strings.Builder
+		buf := make([]byte, 4096)
+		for {
+			n, err := r.Read(buf)
+			if n > 0 {
+				sb.Write(buf[:n])
+			}
+			if err != nil {
+				break
+			}
+		}
+		r.Close()
+		done <- sb.String()
+	}()
+	fn()
+	w.Close()
+	return <-done
+}