feat(run): kernel-native result_schema — validate + in-loop self-repair
tool.Invocation.ResultSchema (JSON Schema, draft 2020-12): the executor validates the single-loop final answer and, on violation, runs up to two bounded corrective rounds on the same conversation (FinalGuard-style extra rounds, but the round's text IS the new candidate); a conforming answer is replaced by the bare validated JSON. Fail-open on compile problems (host pre-dispatch compile is the authoritative gate) and best-effort on persistent violations (last output stands, hosts keep their fallback validator). External $refs fail closed — jsonschema/v6's default loader resolves file:// from local disk, overridden with a deny loader. Audit events: result_schema_round/_valid/_unmet/_skipped. Multi-phase runs unvalidated (FinalGuard's single-loop-only scope). Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,208 @@
|
||||
package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
llm "gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/tool"
|
||||
)
|
||||
|
||||
func resultSchemaExecutor(t *testing.T, m llm.Model, rec RunRecorder) *Executor {
|
||||
t.Helper()
|
||||
ports := Ports{}
|
||||
if rec != nil {
|
||||
ports.Audit = auditFunc{start: func(RunInfo) RunRecorder { return rec }}
|
||||
}
|
||||
return New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: ports,
|
||||
})
|
||||
}
|
||||
|
||||
var personSchema = json.RawMessage(`{"type":"object","required":["name"],"properties":{"name":{"type":"string"}},"additionalProperties":false}`)
|
||||
|
||||
// TestResultSchema_ConformingAnswerPassesThrough — a first-try conforming
|
||||
// answer (fence-wrapped, even) is replaced by the bare JSON with no extra
|
||||
// model rounds.
|
||||
func TestResultSchema_ConformingAnswerPassesThrough(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("Here you go:\n```json\n{\"name\": \"mort\"}\n```"))
|
||||
m, _ := fp.Model("m")
|
||||
|
||||
ex := resultSchemaExecutor(t, m, nil)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs1", CallerID: "c", ResultSchema: personSchema}, "who are you")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != `{"name": "mort"}` {
|
||||
t.Fatalf("output = %q, want the bare validated JSON", res.Output)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 1 {
|
||||
t.Fatalf("model calls = %d, want 1 (no corrective round)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResultSchema_CorrectiveRoundRepairs — a non-conforming first answer
|
||||
// triggers one corrective round whose text becomes the run output.
|
||||
func TestResultSchema_CorrectiveRoundRepairs(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.Reply(`{"name": 42}`), // wrong type
|
||||
fake.Reply(`{"name": "fixed"}`), // corrective round conforms
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
rec := &eventRecorder{}
|
||||
|
||||
ex := resultSchemaExecutor(t, m, rec)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs2", CallerID: "c", ResultSchema: personSchema}, "go")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != `{"name": "fixed"}` {
|
||||
t.Fatalf("output = %q, want the corrected JSON", res.Output)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 2 {
|
||||
t.Fatalf("model calls = %d, want 2", n)
|
||||
}
|
||||
if !recHas(rec, "result_schema_round") || !recHas(rec, "result_schema_valid") {
|
||||
t.Fatalf("expected round + valid audit events, got %v", rec.events)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResultSchema_PersistentViolationKeepsLastOutput — rounds exhaust, the
|
||||
// last candidate stands, and result_schema_unmet is recorded (the host's own
|
||||
// validator is the fallback authority).
|
||||
func TestResultSchema_PersistentViolationKeepsLastOutput(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.Reply("not json at all"),
|
||||
fake.Reply("still prose"),
|
||||
fake.Reply("prose forever"),
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
rec := &eventRecorder{}
|
||||
|
||||
ex := resultSchemaExecutor(t, m, rec)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs3", CallerID: "c", ResultSchema: personSchema}, "go")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "prose forever" {
|
||||
t.Fatalf("output = %q, want the last candidate kept", res.Output)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 1+resultSchemaMaxRounds {
|
||||
t.Fatalf("model calls = %d, want %d", n, 1+resultSchemaMaxRounds)
|
||||
}
|
||||
if !recHas(rec, "result_schema_unmet") {
|
||||
t.Fatalf("expected result_schema_unmet, got %v", rec.events)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResultSchema_NoSchemaUnchanged — anchor: without a schema the output
|
||||
// and call count are untouched.
|
||||
func TestResultSchema_NoSchemaUnchanged(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("plain prose answer"))
|
||||
m, _ := fp.Model("m")
|
||||
|
||||
ex := resultSchemaExecutor(t, m, nil)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs4", CallerID: "c"}, "go")
|
||||
|
||||
if res.Err != nil || res.Output != "plain prose answer" {
|
||||
t.Fatalf("output = %q err = %v", res.Output, res.Err)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 1 {
|
||||
t.Fatalf("model calls = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResultSchema_CompileFailureFailsOpen — an uncompilable schema skips
|
||||
// validation (WARN + audit event), never bricks the run.
|
||||
func TestResultSchema_CompileFailureFailsOpen(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("prose"))
|
||||
m, _ := fp.Model("m")
|
||||
rec := &eventRecorder{}
|
||||
|
||||
ex := resultSchemaExecutor(t, m, rec)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs5", CallerID: "c",
|
||||
ResultSchema: json.RawMessage(`{"type": "not-a-type"}`)}, "go")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "prose" {
|
||||
t.Fatalf("output = %q, want untouched", res.Output)
|
||||
}
|
||||
if !recHas(rec, "result_schema_skipped") {
|
||||
t.Fatalf("expected result_schema_skipped, got %v", rec.events)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompileResultSchema_ExternalRefsFailClosed — http(s) AND file $refs
|
||||
// must not resolve (jsonschema/v6's default loader reads local disk).
|
||||
func TestCompileResultSchema_ExternalRefsFailClosed(t *testing.T) {
|
||||
for _, ref := range []string{
|
||||
"https://example.com/schema.json",
|
||||
"file:///etc/passwd",
|
||||
} {
|
||||
raw := json.RawMessage(`{"$ref": "` + ref + `"}`)
|
||||
if _, err := compileResultSchema(raw); err == nil {
|
||||
t.Errorf("$ref %q compiled — external loading must fail closed", ref)
|
||||
} else if !strings.Contains(err.Error(), "refused") && !strings.Contains(err.Error(), "not exist") {
|
||||
// Any error is acceptable as long as it FAILS; the refused
|
||||
// wording just confirms the deny loader was consulted.
|
||||
t.Logf("$ref %q failed with: %v", ref, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractResultJSON_Shapes — extraction tolerance.
|
||||
func TestExtractResultJSON_Shapes(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
{`{"a":1}`, `{"a":1}`, true},
|
||||
{"prefix text {\"a\": {\"b\": \"}\"}} suffix", `{"a": {"b": "}"}}`, true},
|
||||
{"```json\n[1,2]\n```", `[1,2]`, true},
|
||||
{"no json here", "", false},
|
||||
{"broken { \"a\": ", "", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, ok := extractResultJSON(c.in)
|
||||
if ok != c.ok || (ok && string(got) != c.want) {
|
||||
t.Errorf("extract(%q) = (%q, %v), want (%q, %v)", c.in, got, ok, c.want, c.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recHas reports whether the recorder saw an event type.
|
||||
func recHas(r *eventRecorder, name string) bool {
|
||||
for _, e := range r.events {
|
||||
if e == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user