package main import ( "encoding/json" "os" "path/filepath" "strings" "testing" ) func TestClusterFindingsAgreementAndTolerance(t *testing.T) { models := []modelFindings{ {Model: "m1", Verdict: "Blocking issues found", Findings: []outFinding{ {Lens: "security", File: "a.go", Line: 10, Severity: "high", Title: "auth bypass"}, {Lens: "perf", File: "b.go", Line: 5, Severity: "trivial", Title: "tiny nit"}, }}, {Model: "m2", Verdict: "Minor issues", Findings: []outFinding{ {Lens: "security", File: "a.go", Line: 11, Severity: "critical", Title: "auth bypass (crit)"}, // within tolerance of a.go:10 }}, {Model: "m3", Verdict: "Minor issues", Findings: []outFinding{ {Lens: "correctness", File: "a.go", Line: 10, Severity: "medium", Title: "auth bypass"}, }}, } clusters := clusterFindings(models) if len(clusters) != 2 { t.Fatalf("want 2 clusters (a.go:10±, b.go:5), got %d: %+v", len(clusters), clusters) } // First cluster (highest agreement) is the a.go auth one: 3 models, severity // escalated to critical, representative line the smallest (10). c := clusters[0] if len(c.models) != 3 { t.Errorf("want 3 models on the top cluster, got %d", len(c.models)) } if c.severity != "critical" { t.Errorf("want escalated severity critical, got %q", c.severity) } if c.line != 10 { t.Errorf("want representative line 10, got %d", c.line) } if !c.lenses["security"] || !c.lenses["correctness"] { t.Errorf("want union of lenses, got %v", c.lenses) } } func TestRenderConsensusFoldsSingleModelNits(t *testing.T) { models := []modelFindings{ {Model: "m1", Provider: "p", Verdict: "Blocking issues found", Markdown: "m1 detail", Findings: []outFinding{ {Lens: "security", File: "a.go", Line: 10, Severity: "high", Title: "auth bypass"}, {Lens: "perf", File: "b.go", Line: 5, Severity: "trivial", Title: "tiny nit"}, }}, {Model: "m2", Provider: "p", Verdict: "Minor issues", Markdown: "m2 detail", Findings: []outFinding{ {Lens: "security", File: "a.go", Line: 10, Severity: "high", Title: "auth bypass"}, }}, } out := renderConsensus(models, clusterFindings(models)) // Headline table: the agreed finding with a 2/2 badge. if !strings.Contains(out, "2/2") { t.Errorf("expected a 2/2 agreement badge in headline:\n%s", out) } if !strings.Contains(out, "auth bypass") || !strings.Contains(out, "a.go:10") { t.Errorf("headline missing the consensus finding:\n%s", out) } // The lone trivial finding is folded, not in the headline table. if !strings.Contains(out, "single-model finding") { t.Errorf("expected a folded single-model section:\n%s", out) } // Per-model detail is preserved (folded). if !strings.Contains(out, "m1 detail") || !strings.Contains(out, "m2 detail") { t.Errorf("per-model detail not preserved:\n%s", out) } } func TestRenderConsensusHighSeverityLoneFindingStaysHeadline(t *testing.T) { // A single model, single critical finding must still surface in the headline // (not be folded as "low confidence"). models := []modelFindings{ {Model: "solo", Verdict: "Blocking issues found", Markdown: "x", Findings: []outFinding{ {Lens: "security", File: "a.go", Line: 1, Severity: "critical", Title: "rce"}, }}, } out := renderConsensus(models, clusterFindings(models)) headline := out if i := strings.Index(out, "single-model finding"); i >= 0 { headline = out[:i] } if !strings.Contains(headline, "rce") { t.Errorf("lone critical should be in the headline, not folded:\n%s", out) } } func TestClusterSlidingWindowMergesChain(t *testing.T) { // Findings at 10, 13, 16 (each 3 apart) from three models must merge into ONE // cluster — the window slides with the span instead of anchoring at line 10. models := []modelFindings{ {Model: "m1", Findings: []outFinding{{Lens: "x", File: "a.go", Line: 10, Severity: "medium", Title: "t"}}}, {Model: "m2", Findings: []outFinding{{Lens: "x", File: "a.go", Line: 13, Severity: "medium", Title: "t"}}}, {Model: "m3", Findings: []outFinding{{Lens: "x", File: "a.go", Line: 16, Severity: "medium", Title: "t"}}}, } clusters := clusterFindings(models) if len(clusters) != 1 { t.Fatalf("chain 10/13/16 should merge into 1 cluster, got %d", len(clusters)) } if len(clusters[0].models) != 3 { t.Errorf("want 3 models in the merged cluster, got %d", len(clusters[0].models)) } } func TestRenderConsensusExcludesErroredFromDenominator(t *testing.T) { models := []modelFindings{ {Model: "m1", Verdict: "Minor issues", Markdown: "a", Findings: []outFinding{ {Lens: "security", File: "a.go", Line: 9, Severity: "medium", Title: "leak"}}}, {Model: "m2", Verdict: "Minor issues", Markdown: "b", Findings: []outFinding{ {Lens: "security", File: "a.go", Line: 9, Severity: "medium", Title: "leak"}}}, {Model: "broken", Verdict: "reviewer failed", Errored: true, Markdown: "boom"}, } out := renderConsensus(models, clusterFindings(models)) // Denominator is the 2 effective models, not 3; the failure is noted. if !strings.Contains(out, "2/2") { t.Errorf("errored model must be excluded from the denominator (want 2/2):\n%s", out) } if !strings.Contains(out, "1 failed") { t.Errorf("expected a '1 failed' note:\n%s", out) } if !strings.Contains(out, "reviewer failed") { t.Errorf("errored model should still appear (folded) as failed:\n%s", out) } } func TestRenderConsensusLoneHighFolds(t *testing.T) { // A single-model HIGH (not critical) folds — only consensus or a lone CRITICAL // earns the headline, so a lone Blocking-lens finding doesn't reintroduce noise. models := []modelFindings{ {Model: "solo", Verdict: "Blocking issues found", Markdown: "x", Findings: []outFinding{ {Lens: "security", File: "a.go", Line: 1, Severity: "high", Title: "maybe-bug"}}}, } out := renderConsensus(models, clusterFindings(models)) head := out if i := strings.Index(out, "single-model finding"); i >= 0 { head = out[:i] } if strings.Contains(head, "maybe-bug") { t.Errorf("a lone HIGH should fold, not headline:\n%s", out) } } func TestWriteAndConsolidateRoundTrip(t *testing.T) { dir := t.TempDir() // Two model artifacts on disk. write := func(name string, mf modelFindings) { data, _ := json.Marshal(mf) if err := os.WriteFile(filepath.Join(dir, name), data, 0o644); err != nil { t.Fatal(err) } } write("m1.json", modelFindings{Model: "m1", Provider: "ollama", Verdict: "Minor issues", Markdown: "md1", Findings: []outFinding{{Lens: "security", File: "x.go", Line: 3, Severity: "medium", Title: "leak"}}}) write("m2.json", modelFindings{Model: "m2", Provider: "ollama", Verdict: "Minor issues", Markdown: "md2", Findings: []outFinding{{Lens: "security", File: "x.go", Line: 3, Severity: "high", Title: "leak"}}}) // A junk file must be skipped, not crash consolidation. if err := os.WriteFile(filepath.Join(dir, "notes.txt"), []byte("ignore me"), 0o644); err != nil { t.Fatal(err) } t.Setenv("GADFLY_CONSOLIDATE_DIR", dir) // runConsolidate prints to stdout; capture it. out := captureStdout(t, func() { if err := runConsolidate(); err != nil { t.Fatalf("runConsolidate: %v", err) } }) if !strings.HasPrefix(strings.TrimSpace(out), consensusMarker) { t.Errorf("consolidated output must lead with the marker:\n%s", out) } if !strings.Contains(out, "2/2") || !strings.Contains(out, "x.go:3") { t.Errorf("expected the agreed x.go:3 finding at 2/2:\n%s", out) } } func TestRunConsolidateEmptyDirErrors(t *testing.T) { t.Setenv("GADFLY_CONSOLIDATE_DIR", t.TempDir()) if err := runConsolidate(); err == nil { t.Error("want an error for an empty consolidate dir (entrypoint falls back)") } } // captureStdout redirects os.Stdout for the duration of fn and returns what was // written. func captureStdout(t *testing.T, fn func()) string { t.Helper() orig := os.Stdout r, w, err := os.Pipe() if err != nil { t.Fatal(err) } os.Stdout = w defer func() { os.Stdout = orig }() // restore even if fn panics done := make(chan string) go func() { var sb strings.Builder buf := make([]byte, 4096) for { n, err := r.Read(buf) if n > 0 { sb.Write(buf[:n]) } if err != nil { break } } r.Close() done <- sb.String() }() fn() w.Close() return <-done }