fix(review): result_schema gadfly findings + go mod tidy
executus CI / test (pull_request) Successful in 50s
executus CI / test (pull_request) Successful in 50s
- Extracted applyResultSchema (panic-isolated like safeFinalNudge — a validator bug must never convert a successful run into a panic error) - Blank corrective-round reply no longer clobbers the prior candidate - result_schema_valid now fires on first-try conforming answers too - Multi-phase runs log WARN + result_schema_skipped instead of a silent drop - errors.As instead of the hand-rolled assertion; redundant guard removed; go.mod/go.sum tidied (the CI failure) Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
+107
-10
@@ -1,13 +1,21 @@
|
||||
package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
|
||||
llm "gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/tool"
|
||||
)
|
||||
|
||||
// result_schema.go — kernel-native structured output for single-loop runs.
|
||||
@@ -75,7 +83,7 @@ func extractResultJSON(s string) (json.RawMessage, bool) {
|
||||
if len(s) > resultSchemaExtractCap {
|
||||
s = s[:resultSchemaExtractCap]
|
||||
}
|
||||
if json.Valid([]byte(s)) && len(s) > 0 {
|
||||
if json.Valid([]byte(s)) {
|
||||
return json.RawMessage(s), true
|
||||
}
|
||||
// Fenced block: ```json ... ``` or bare ``` ... ```.
|
||||
@@ -150,7 +158,7 @@ func validateResultJSON(sch *jsonschema.Schema, doc json.RawMessage) []string {
|
||||
return nil
|
||||
}
|
||||
var ve *jsonschema.ValidationError
|
||||
if ok := asValidationError(err, &ve); !ok {
|
||||
if !errors.As(err, &ve) {
|
||||
return []string{err.Error()}
|
||||
}
|
||||
var out []string
|
||||
@@ -162,14 +170,6 @@ func validateResultJSON(sch *jsonschema.Schema, doc json.RawMessage) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func asValidationError(err error, target **jsonschema.ValidationError) bool {
|
||||
ve, ok := err.(*jsonschema.ValidationError)
|
||||
if ok {
|
||||
*target = ve
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
// flattenValidationError collects leaf causes as "/path: message" lines.
|
||||
func flattenValidationError(ve *jsonschema.ValidationError, out *[]string) {
|
||||
if len(ve.Causes) == 0 {
|
||||
@@ -185,6 +185,103 @@ func flattenValidationError(ve *jsonschema.ValidationError, out *[]string) {
|
||||
}
|
||||
}
|
||||
|
||||
// resultSchemaDeps bundles the per-run state applyResultSchema needs from
|
||||
// the executor's single-loop path.
|
||||
type resultSchemaDeps struct {
|
||||
inv tool.Invocation
|
||||
ra RunnableAgent
|
||||
runRes *agent.Result
|
||||
rec RunRecorder
|
||||
model llm.Model
|
||||
toolbox *llm.Toolbox
|
||||
sharedOpts []agent.Option
|
||||
obs func(agent.Step)
|
||||
steer func() []llm.Message
|
||||
noteCheckpoint func(llm.Message)
|
||||
}
|
||||
|
||||
// applyResultSchema runs the validate + corrective-round loop, mutating
|
||||
// d.runRes in place. Panic-isolated like safeFinalNudge: a bug in the
|
||||
// validator (or a pathological schema) must never convert a successful
|
||||
// run into a panic error.
|
||||
func (e *Executor) applyResultSchema(runCtx context.Context, d resultSchemaDeps) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
slog.Error("run: result_schema validation panicked; keeping run result",
|
||||
"run_id", d.inv.RunID, "panic", r)
|
||||
}
|
||||
}()
|
||||
sch, cerr := compileResultSchema(d.inv.ResultSchema)
|
||||
if cerr != nil {
|
||||
slog.Warn("run: result_schema failed to compile; skipping kernel validation",
|
||||
"run_id", d.inv.RunID, "error", cerr)
|
||||
if d.rec != nil {
|
||||
d.rec.LogEvent("result_schema_skipped", map[string]any{"error": cerr.Error()})
|
||||
}
|
||||
return
|
||||
}
|
||||
for round := 0; ; round++ {
|
||||
var errs []string
|
||||
doc, ok := extractResultJSON(d.runRes.Output)
|
||||
if ok {
|
||||
errs = validateResultJSON(sch, doc)
|
||||
if errs == nil {
|
||||
// Conforming: the bare validated document IS the answer.
|
||||
d.runRes.Output = string(doc)
|
||||
if d.rec != nil {
|
||||
d.rec.LogEvent("result_schema_valid", map[string]any{"rounds": round})
|
||||
}
|
||||
return
|
||||
}
|
||||
} else {
|
||||
errs = []string{"no JSON document found in output"}
|
||||
}
|
||||
if round >= resultSchemaMaxRounds {
|
||||
// Best effort exhausted — keep the last candidate; the host's
|
||||
// own validator decides what to do with it.
|
||||
if d.rec != nil {
|
||||
d.rec.LogEvent("result_schema_unmet", map[string]any{
|
||||
"rounds": round, "validation_errors": errs,
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
correction := resultSchemaCorrection(d.inv.ResultSchema, errs)
|
||||
if d.rec != nil {
|
||||
d.rec.LogEvent("result_schema_round", map[string]any{
|
||||
"round": round + 1, "validation_errors": errs,
|
||||
})
|
||||
}
|
||||
if d.noteCheckpoint != nil {
|
||||
d.noteCheckpoint(llm.UserText(correction))
|
||||
}
|
||||
roundOpts := append([]agent.Option{
|
||||
agent.WithToolbox(d.toolbox),
|
||||
agent.WithMaxSteps(resultSchemaRoundMaxSteps),
|
||||
agent.WithStepObserver(d.obs),
|
||||
}, d.sharedOpts...)
|
||||
roundAg := agent.New(d.model, e.systemPrompt(d.ra), roundOpts...)
|
||||
roundRes, roundErr := roundAg.Run(runCtx, correction,
|
||||
agent.WithSteer(d.steer), agent.WithHistory(d.runRes.Messages))
|
||||
if roundRes != nil {
|
||||
roundUsage := roundRes.Usage
|
||||
d.runRes.Usage.Add(roundUsage)
|
||||
}
|
||||
if roundErr != nil || roundRes == nil {
|
||||
slog.Warn("run: result_schema corrective round failed; keeping last output",
|
||||
"run_id", d.inv.RunID, "error", roundErr)
|
||||
return
|
||||
}
|
||||
// The round's text is the NEW candidate (contrast FinalGuard, whose
|
||||
// extra-round text is discarded meta-commentary) — but a BLANK round
|
||||
// reply must not clobber the prior non-empty candidate.
|
||||
if strings.TrimSpace(roundRes.Output) != "" {
|
||||
d.runRes.Output = roundRes.Output
|
||||
}
|
||||
d.runRes.Messages = roundRes.Messages
|
||||
}
|
||||
}
|
||||
|
||||
// resultSchemaCorrection builds the corrective user turn for one round.
|
||||
func resultSchemaCorrection(raw json.RawMessage, errs []string) string {
|
||||
return fmt.Sprintf(`Your final answer must be a SINGLE JSON document conforming to this JSON Schema (no prose, no code fences):
|
||||
|
||||
Reference in New Issue
Block a user