feat(run): kernel-native result_schema — validate + in-loop self-repair
executus CI / test (pull_request) Failing after 38s
Gadfly review (reusable) / review (pull_request) Successful in 11m34s
Adversarial Review (Gadfly) / review (pull_request) Successful in 11m34s

tool.Invocation.ResultSchema (JSON Schema, draft 2020-12): the executor
validates the single-loop final answer and, on violation, runs up to two
bounded corrective rounds on the same conversation (FinalGuard-style
extra rounds, but the round's text IS the new candidate); a conforming
answer is replaced by the bare validated JSON. Fail-open on compile
problems (host pre-dispatch compile is the authoritative gate) and
best-effort on persistent violations (last output stands, hosts keep
their fallback validator). External $refs fail closed — jsonschema/v6's
default loader resolves file:// from local disk, overridden with a deny
loader. Audit events: result_schema_round/_valid/_unmet/_skipped.
Multi-phase runs unvalidated (FinalGuard's single-loop-only scope).

Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
2026-07-17 13:17:43 -04:00
co-authored by Claude Fable 5
parent 46e4eb3da6
commit b4080a8a6e
6 changed files with 496 additions and 0 deletions
+72
View File
@@ -492,6 +492,78 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
runRes, runErr = runAgent(runCtx, ag, input, inv.Images, agent.WithSteer(steer))
}
// ResultSchema (optional): validate the final answer and let the model
// self-repair on the same conversation before the host sees it. Runs
// BEFORE FinalGuard so the delivery check inspects the answer that will
// actually be returned. Fail-open on compile problems; best-effort on
// persistent violations (the host's fallback validator is authoritative).
if runErr == nil && runRes != nil && len(inv.ResultSchema) > 0 {
if sch, cerr := compileResultSchema(inv.ResultSchema); cerr != nil {
slog.Warn("run: result_schema failed to compile; skipping kernel validation",
"run_id", inv.RunID, "error", cerr)
if rec != nil {
rec.LogEvent("result_schema_skipped", map[string]any{"error": cerr.Error()})
}
} else {
for round := 0; ; round++ {
var errs []string
doc, ok := extractResultJSON(runRes.Output)
if ok {
errs = validateResultJSON(sch, doc)
if errs == nil {
// Conforming: the bare validated document IS the answer.
runRes.Output = string(doc)
if rec != nil && round > 0 {
rec.LogEvent("result_schema_valid", map[string]any{"rounds": round})
}
break
}
} else {
errs = []string{"no JSON document found in output"}
}
if round >= resultSchemaMaxRounds {
// Best effort exhausted — keep the last candidate; the
// host's own validator decides what to do with it.
if rec != nil {
rec.LogEvent("result_schema_unmet", map[string]any{
"rounds": round, "validation_errors": errs,
})
}
break
}
correction := resultSchemaCorrection(inv.ResultSchema, errs)
if rec != nil {
rec.LogEvent("result_schema_round", map[string]any{
"round": round + 1, "validation_errors": errs,
})
}
if noteCheckpoint != nil {
noteCheckpoint(llm.UserText(correction))
}
roundOpts := append([]agent.Option{
agent.WithToolbox(toolbox),
agent.WithMaxSteps(resultSchemaRoundMaxSteps),
agent.WithStepObserver(obs),
}, sharedOpts...)
roundAg := agent.New(model, e.systemPrompt(ra), roundOpts...)
roundRes, roundErr := roundAg.Run(runCtx, correction,
agent.WithSteer(steer), agent.WithHistory(runRes.Messages))
if roundRes != nil {
runRes.Usage.Add(roundRes.Usage)
}
if roundErr != nil || roundRes == nil {
slog.Warn("run: result_schema corrective round failed; keeping last output",
"run_id", inv.RunID, "error", roundErr)
break
}
// The round's text is the NEW candidate (contrast FinalGuard,
// whose extra-round text is discarded meta-commentary).
runRes.Output = roundRes.Output
runRes.Messages = roundRes.Messages
}
}
}
// FinalGuard (optional): the run finished successfully — before the result
// is finalized, let the host inspect it for undelivered work (the
// announce-then-stop failure: an artifact rendered into run storage, a