package run import ( "context" "encoding/json" "strings" "testing" llm "gitea.stevedudenhoeffer.com/steve/majordomo/llm" "gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake" "gitea.stevedudenhoeffer.com/steve/executus/tool" ) func resultSchemaExecutor(t *testing.T, m llm.Model, rec RunRecorder) *Executor { t.Helper() ports := Ports{} if rec != nil { ports.Audit = auditFunc{start: func(RunInfo) RunRecorder { return rec }} } return New(Config{ Registry: tool.NewRegistry(), Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil }, Ports: ports, }) } var personSchema = json.RawMessage(`{"type":"object","required":["name"],"properties":{"name":{"type":"string"}},"additionalProperties":false}`) // TestResultSchema_ConformingAnswerPassesThrough — a first-try conforming // answer (fence-wrapped, even) is replaced by the bare JSON with no extra // model rounds. func TestResultSchema_ConformingAnswerPassesThrough(t *testing.T) { fp := fake.New("fake") fp.Enqueue("m", fake.Reply("Here you go:\n```json\n{\"name\": \"mort\"}\n```")) m, _ := fp.Model("m") ex := resultSchemaExecutor(t, m, nil) res := ex.Run(context.Background(), RunnableAgent{Name: "g", ModelTier: "m"}, tool.Invocation{RunID: "r-rs1", CallerID: "c", ResultSchema: personSchema}, "who are you") if res.Err != nil { t.Fatalf("run error: %v", res.Err) } if res.Output != `{"name": "mort"}` { t.Fatalf("output = %q, want the bare validated JSON", res.Output) } if n := fp.CallCount("m"); n != 1 { t.Fatalf("model calls = %d, want 1 (no corrective round)", n) } } // TestResultSchema_CorrectiveRoundRepairs — a non-conforming first answer // triggers one corrective round whose text becomes the run output. func TestResultSchema_CorrectiveRoundRepairs(t *testing.T) { fp := fake.New("fake") fp.Enqueue("m", fake.Reply(`{"name": 42}`), // wrong type fake.Reply(`{"name": "fixed"}`), // corrective round conforms ) m, _ := fp.Model("m") rec := &eventRecorder{} ex := resultSchemaExecutor(t, m, rec) res := ex.Run(context.Background(), RunnableAgent{Name: "g", ModelTier: "m"}, tool.Invocation{RunID: "r-rs2", CallerID: "c", ResultSchema: personSchema}, "go") if res.Err != nil { t.Fatalf("run error: %v", res.Err) } if res.Output != `{"name": "fixed"}` { t.Fatalf("output = %q, want the corrected JSON", res.Output) } if n := fp.CallCount("m"); n != 2 { t.Fatalf("model calls = %d, want 2", n) } if !recHas(rec, "result_schema_round") || !recHas(rec, "result_schema_valid") { t.Fatalf("expected round + valid audit events, got %v", rec.events) } } // TestResultSchema_PersistentViolationKeepsLastOutput — rounds exhaust, the // last candidate stands, and result_schema_unmet is recorded (the host's own // validator is the fallback authority). func TestResultSchema_PersistentViolationKeepsLastOutput(t *testing.T) { fp := fake.New("fake") fp.Enqueue("m", fake.Reply("not json at all"), fake.Reply("still prose"), fake.Reply("prose forever"), ) m, _ := fp.Model("m") rec := &eventRecorder{} ex := resultSchemaExecutor(t, m, rec) res := ex.Run(context.Background(), RunnableAgent{Name: "g", ModelTier: "m"}, tool.Invocation{RunID: "r-rs3", CallerID: "c", ResultSchema: personSchema}, "go") if res.Err != nil { t.Fatalf("run error: %v", res.Err) } if res.Output != "prose forever" { t.Fatalf("output = %q, want the last candidate kept", res.Output) } if n := fp.CallCount("m"); n != 1+resultSchemaMaxRounds { t.Fatalf("model calls = %d, want %d", n, 1+resultSchemaMaxRounds) } if !recHas(rec, "result_schema_unmet") { t.Fatalf("expected result_schema_unmet, got %v", rec.events) } } // TestResultSchema_NoSchemaUnchanged — anchor: without a schema the output // and call count are untouched. func TestResultSchema_NoSchemaUnchanged(t *testing.T) { fp := fake.New("fake") fp.Enqueue("m", fake.Reply("plain prose answer")) m, _ := fp.Model("m") ex := resultSchemaExecutor(t, m, nil) res := ex.Run(context.Background(), RunnableAgent{Name: "g", ModelTier: "m"}, tool.Invocation{RunID: "r-rs4", CallerID: "c"}, "go") if res.Err != nil || res.Output != "plain prose answer" { t.Fatalf("output = %q err = %v", res.Output, res.Err) } if n := fp.CallCount("m"); n != 1 { t.Fatalf("model calls = %d, want 1", n) } } // TestResultSchema_CompileFailureFailsOpen — an uncompilable schema skips // validation (WARN + audit event), never bricks the run. func TestResultSchema_CompileFailureFailsOpen(t *testing.T) { fp := fake.New("fake") fp.Enqueue("m", fake.Reply("prose")) m, _ := fp.Model("m") rec := &eventRecorder{} ex := resultSchemaExecutor(t, m, rec) res := ex.Run(context.Background(), RunnableAgent{Name: "g", ModelTier: "m"}, tool.Invocation{RunID: "r-rs5", CallerID: "c", ResultSchema: json.RawMessage(`{"type": "not-a-type"}`)}, "go") if res.Err != nil { t.Fatalf("run error: %v", res.Err) } if res.Output != "prose" { t.Fatalf("output = %q, want untouched", res.Output) } if !recHas(rec, "result_schema_skipped") { t.Fatalf("expected result_schema_skipped, got %v", rec.events) } } // TestCompileResultSchema_ExternalRefsFailClosed — http(s) AND file $refs // must not resolve (jsonschema/v6's default loader reads local disk). func TestCompileResultSchema_ExternalRefsFailClosed(t *testing.T) { for _, ref := range []string{ "https://example.com/schema.json", "file:///etc/passwd", } { raw := json.RawMessage(`{"$ref": "` + ref + `"}`) if _, err := compileResultSchema(raw); err == nil { t.Errorf("$ref %q compiled — external loading must fail closed", ref) } else if !strings.Contains(err.Error(), "refused") && !strings.Contains(err.Error(), "not exist") { // Any error is acceptable as long as it FAILS; the refused // wording just confirms the deny loader was consulted. t.Logf("$ref %q failed with: %v", ref, err) } } } // TestExtractResultJSON_Shapes — extraction tolerance. func TestExtractResultJSON_Shapes(t *testing.T) { cases := []struct { in string want string ok bool }{ {`{"a":1}`, `{"a":1}`, true}, {"prefix text {\"a\": {\"b\": \"}\"}} suffix", `{"a": {"b": "}"}}`, true}, {"```json\n[1,2]\n```", `[1,2]`, true}, {"no json here", "", false}, {"broken { \"a\": ", "", false}, } for _, c := range cases { got, ok := extractResultJSON(c.in) if ok != c.ok || (ok && string(got) != c.want) { t.Errorf("extract(%q) = (%q, %v), want (%q, %v)", c.in, got, ok, c.want, c.ok) } } } // recHas reports whether the recorder saw an event type. func recHas(r *eventRecorder, name string) bool { for _, e := range r.events { if e == name { return true } } return false }