Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
55a8fb8866 | ||
|
|
13be3022fd | ||
|
|
25563e1f21 | ||
|
|
a6b185985a | ||
|
|
b4080a8a6e | ||
|
|
46e4eb3da6 | ||
|
|
9c6b819f71 | ||
|
|
cc3739494c | ||
|
|
17e329f935 | ||
|
|
42206f0af0 | ||
|
|
7a074d8fa2 | ||
|
|
007b7539c2 | ||
|
|
8ecdadf8b8 |
@@ -42,7 +42,7 @@ jobs:
|
||||
# and cache the reusable-workflow ref, so a moved v1 tag keeps resolving to the
|
||||
# stale cached copy. A unique sha forces a cache miss → fresh fetch. Bump this
|
||||
# sha to adopt central swarm changes.
|
||||
uses: steve/gadfly/.gitea/workflows/review-reusable.yml@5007597cf921dc3f0a83c708878facfe65fd8e8b
|
||||
uses: steve/gadfly/.gitea/workflows/review-reusable.yml@8eb0265e759e8c37ade8eaa8cb4b3aa9c7a5a169
|
||||
# Least privilege: forward only the review secrets (not `secrets: inherit`,
|
||||
# which would expose every repo secret). GITEA_TOKEN is the automatic token.
|
||||
secrets:
|
||||
@@ -53,3 +53,6 @@ jobs:
|
||||
with:
|
||||
# Consumer-specific allow-list; everything else is inherited.
|
||||
allowed_users: "steve,fizi,dazed"
|
||||
# Gitea >= 1.27 does not propagate dispatch inputs into a called workflow's
|
||||
# github.event — thread the PR number explicitly (empty on non-dispatch events).
|
||||
pr_number: ${{ github.event.inputs.pr_number }}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# gifsmith — a portable, focused render agent that makes animated GIFs/MP4s via
|
||||
# the `gif` skill pack. Shipped by executus (agentbuiltins), run by any host that
|
||||
# provides tools with these names, a `thinking` model tier, and the `gif` pack.
|
||||
# Nothing here is host-specific — the names are the contract the host binds.
|
||||
name: gifsmith
|
||||
description: >-
|
||||
Makes a funny animated GIF (or an MP4 when the piece is long or a GIF is too
|
||||
big) from a description, via the gif skill pack. A single-purpose render agent
|
||||
— use it for any request to draw/animate/gif something, including multi-minute
|
||||
bits about people or things that happened.
|
||||
model_tier: thinking
|
||||
system_prompt: |
|
||||
You make funny animated GIFs and MP4s from a description — often caricatures of
|
||||
the people in the channel or a bit about something that happened. Work by
|
||||
calling tools; do NOT introduce yourself or list capabilities.
|
||||
|
||||
Load the `gif` skill FIRST: call skill_use with name `gif` to get the full
|
||||
recipe (scene/cast planning, the code_exec workspace rules, the bundled encode
|
||||
helper, and the GIF-vs-MP4 size/length decision), then follow it exactly to
|
||||
render and deliver the result. The skill also bundles an encode helper that
|
||||
picks GIF vs MP4 and guarantees a Discord-playable MP4 — use it, don't hand-roll
|
||||
the encode.
|
||||
|
||||
Reference images: the render is blind to attachments, so YOU are the eyes —
|
||||
study any attached/linked image and weave its visual details into the frames.
|
||||
If you can't make it out, proceed from the words.
|
||||
low_level_tools:
|
||||
- code_exec
|
||||
- image_describe
|
||||
- send_attachments
|
||||
- file_get_metadata
|
||||
- file_save
|
||||
- think
|
||||
skill_packs:
|
||||
- gif
|
||||
execution_lane: animate
|
||||
max_iterations: 50
|
||||
max_tool_calls: 80
|
||||
max_runtime_seconds: 1800
|
||||
critic_enabled: true
|
||||
default_emoji: "🎬"
|
||||
state_react:
|
||||
__start__: "🎬"
|
||||
code_exec: "🐍"
|
||||
image_describe: "🖼️"
|
||||
think: "🧠"
|
||||
send_attachments: "📎"
|
||||
__end__: "✅"
|
||||
__error__: "❌"
|
||||
@@ -0,0 +1,24 @@
|
||||
// Package agentbuiltins ships executus's canonical builtin agent definitions as
|
||||
// an embedded filesystem. They are portable persona manifests
|
||||
// (agents/<name>/agent.yml): each references tool NAMES, a model-tier NAME, and
|
||||
// skill-pack names — the host binds those to implementations. Nothing here
|
||||
// imports a host or a battery, so any executus consumer can seed these via
|
||||
// persona.LoadBuiltinAgents (or its own loader that reads the same schema):
|
||||
//
|
||||
// persona.LoadBuiltinAgents(ctx, store, agentbuiltins.FS(), skillChecker)
|
||||
//
|
||||
// Ships:
|
||||
// - gifsmith — a focused GIF/MP4 render agent that uses the `gif` skill pack.
|
||||
package agentbuiltins
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
)
|
||||
|
||||
//go:embed agents
|
||||
var embedded embed.FS
|
||||
|
||||
// FS returns the builtin agents tree, rooted so that a loader finds each
|
||||
// definition at agents/<name>/agent.yml (the layout LoadBuiltinAgents expects).
|
||||
func FS() fs.FS { return embedded }
|
||||
@@ -0,0 +1,42 @@
|
||||
package agentbuiltins_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/agentbuiltins"
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/persona"
|
||||
)
|
||||
|
||||
// TestGifsmithLoads proves executus's shipped gifsmith manifest flows through
|
||||
// the persona loader and lowers into a RunnableAgent carrying the gif pack — the
|
||||
// path a host uses to dogfood it.
|
||||
func TestGifsmithLoads(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := persona.NewMemory()
|
||||
n, err := persona.LoadBuiltinAgents(ctx, store, agentbuiltins.FS(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n < 1 {
|
||||
t.Fatalf("expected gifsmith seeded, got %d", n)
|
||||
}
|
||||
a, err := store.GetAgentByName(ctx, persona.BuiltinAgentOwnerID, "gifsmith")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(a.SkillPacks) != 1 || a.SkillPacks[0] != "gif" {
|
||||
t.Errorf("skill_packs = %v", a.SkillPacks)
|
||||
}
|
||||
if a.ModelTier != "thinking" {
|
||||
t.Errorf("model_tier = %q (want a portable tier name)", a.ModelTier)
|
||||
}
|
||||
if !slices.Contains(a.LowLevelTools, "code_exec") || !slices.Contains(a.LowLevelTools, "send_attachments") {
|
||||
t.Errorf("low_level_tools missing render/deliver tools: %v", a.LowLevelTools)
|
||||
}
|
||||
// The pack must survive the lowering the executor consumes.
|
||||
if ra := a.ToRunnable(); len(ra.SkillPacks) != 1 || ra.SkillPacks[0] != "gif" {
|
||||
t.Errorf("RunnableAgent.SkillPacks = %v", ra.SkillPacks)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ require (
|
||||
gitea.stevedudenhoeffer.com/steve/majordomo v0.0.0-20260626223738-1fd7109a42f3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
golang.org/x/crypto v0.53.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
@@ -13,6 +13,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
@@ -52,6 +54,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
|
||||
@@ -86,6 +86,9 @@ type Agent struct {
|
||||
SkillPalette []string // skill IDs/names
|
||||
SubAgentPalette []string // agent IDs/names
|
||||
LowLevelTools []string // skilltools registry names
|
||||
// SkillPacks names SKILL.md skill-pack subscriptions activated for a run via
|
||||
// run.Ports.SkillPacks (catalog folded into the prompt + a skill_use loader).
|
||||
SkillPacks []string
|
||||
|
||||
// Personalization (Phase 5 reads these). Each layer name maps to
|
||||
// a registered PersonalizationProvider that returns text appended
|
||||
|
||||
@@ -291,6 +291,9 @@ func resolveExtends(child, parent *Agent) {
|
||||
if child.LowLevelTools == nil {
|
||||
child.LowLevelTools = parent.LowLevelTools
|
||||
}
|
||||
if child.SkillPacks == nil {
|
||||
child.SkillPacks = parent.SkillPacks
|
||||
}
|
||||
if child.PersonalizationSources == nil {
|
||||
child.PersonalizationSources = parent.PersonalizationSources
|
||||
}
|
||||
@@ -456,6 +459,7 @@ type builtinAgentManifest struct {
|
||||
SkillPalette []string `yaml:"skill_palette"`
|
||||
SubAgentPalette []string `yaml:"sub_agent_palette"`
|
||||
LowLevelTools []string `yaml:"low_level_tools"`
|
||||
SkillPacks []string `yaml:"skill_packs"`
|
||||
|
||||
PersonalizationSources []string `yaml:"personalization_sources"`
|
||||
|
||||
@@ -562,6 +566,7 @@ func decodeAgentManifest(data []byte) (*Agent, error) {
|
||||
SkillPalette: m.SkillPalette,
|
||||
SubAgentPalette: m.SubAgentPalette,
|
||||
LowLevelTools: m.LowLevelTools,
|
||||
SkillPacks: m.SkillPacks,
|
||||
PersonalizationSources: m.PersonalizationSources,
|
||||
Schedule: strings.TrimSpace(m.Schedule),
|
||||
WebhookIPAllowlist: allowlist,
|
||||
|
||||
@@ -18,6 +18,7 @@ func (a *Agent) ToRunnable() run.RunnableAgent {
|
||||
LowLevelTools: a.LowLevelTools,
|
||||
SkillPalette: a.SkillPalette,
|
||||
SubAgentPalette: a.SubAgentPalette,
|
||||
SkillPacks: a.SkillPacks,
|
||||
Critic: run.CriticConfig{
|
||||
Enabled: a.CriticEnabled,
|
||||
BackstopMultiplier: a.CriticBackstopMultiplier,
|
||||
|
||||
@@ -37,6 +37,15 @@ type RunnableAgent struct {
|
||||
MaxIterations int
|
||||
MaxRuntime time.Duration
|
||||
|
||||
// MaxToolCalls caps the TOTAL number of tool calls across the whole run,
|
||||
// independent of the step ceiling (one step can carry several parallel tool
|
||||
// calls). 0 (or negative) = unlimited — the backward-compatible default, so
|
||||
// a run that never sets it is unaffected. Enforced at step granularity: the
|
||||
// loop stops after the step in which the cumulative tool-call count reaches
|
||||
// the cap, surfacing ErrMaxToolCalls. Single-loop runs only today (phased
|
||||
// pipelines are a follow-up).
|
||||
MaxToolCalls int
|
||||
|
||||
// LowLevelTools are tool-registry names the run may call directly.
|
||||
// SkillPalette / SubAgentPalette name saved skills / sub-agents exposed as
|
||||
// skill__<name> / agent__<name> delegation tools, resolved through
|
||||
@@ -44,6 +53,11 @@ type RunnableAgent struct {
|
||||
LowLevelTools []string
|
||||
SkillPalette []string
|
||||
SubAgentPalette []string
|
||||
// SkillPacks names SKILL.md skill-pack subscriptions activated for the run
|
||||
// via Ports.SkillPacks: each pack's name+description joins a catalog folded
|
||||
// into the system prompt, and a skill_use tool loads a pack's body on demand
|
||||
// (progressive disclosure). nil Ports.SkillPacks => inert.
|
||||
SkillPacks []string
|
||||
|
||||
// Phases optionally model a multi-step pipeline (each phase its own prompt
|
||||
// + tier + tools). An empty slice is a single-phase run — the common case.
|
||||
|
||||
+28
-10
@@ -110,17 +110,35 @@ func (b *criticBinding) recordToolStart(name, args string) {
|
||||
}
|
||||
}
|
||||
|
||||
// maxStepsOption returns the agent step-ceiling Option. With no critic it's a
|
||||
// fixed WithMaxSteps(base); with a critic it's a DYNAMIC WithMaxStepsFunc that
|
||||
// polls the handle each step (so the critic can raise a long run's budget),
|
||||
// falling back to base when the handle defers (MaxSteps() <= 0).
|
||||
func (b *criticBinding) maxStepsOption(base int) agent.Option {
|
||||
if b == nil {
|
||||
return agent.WithMaxSteps(base)
|
||||
}
|
||||
// stepCeilingOption returns the agent step-ceiling Option, folding the run's
|
||||
// tool-call cap into the (optionally critic-driven) step ceiling. Priority:
|
||||
//
|
||||
// 1. tool-call cap hit (maxCalls > 0 && *toolCalls >= maxCalls): return the
|
||||
// completed-step count so majordomo's `stepIdx < ceiling` check fails on the
|
||||
// next iteration and the loop exits (the executor relabels the resulting
|
||||
// ErrMaxSteps to ErrMaxToolCalls). Never returns <= 0 — majordomo would fall
|
||||
// back to its own static ceiling — so a cap hit before any step completes
|
||||
// clamps to 1.
|
||||
// 2. the critic's DYNAMIC ceiling, when a critic is bound and raises a long
|
||||
// run's budget mid-flight (b.h.MaxSteps() > 0).
|
||||
// 3. the static base (MaxIterations).
|
||||
//
|
||||
// With maxCalls <= 0 and no critic this is exactly the old WithMaxSteps(base):
|
||||
// an unlimited-tool-call run is a true no-op. nil-safe on the binding (b == nil
|
||||
// skips the critic branch). toolCalls/steps are read live from the step
|
||||
// observer's counters (same goroutine as this poll — see executor.go).
|
||||
func (b *criticBinding) stepCeilingOption(base, maxCalls int, toolCalls, steps *int) agent.Option {
|
||||
return agent.WithMaxStepsFunc(func() int {
|
||||
if n := b.h.MaxSteps(); n > 0 {
|
||||
return n
|
||||
if maxCalls > 0 && toolCalls != nil && *toolCalls >= maxCalls {
|
||||
if steps != nil && *steps > 0 {
|
||||
return *steps
|
||||
}
|
||||
return 1
|
||||
}
|
||||
if b != nil {
|
||||
if n := b.h.MaxSteps(); n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return base
|
||||
})
|
||||
|
||||
+191
-4
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
|
||||
@@ -40,6 +41,17 @@ type Defaults struct {
|
||||
// absolute max. Never shorter than the run's MaxRuntime. Non-critic runs ignore
|
||||
// it (they keep the literal MaxRuntime kill).
|
||||
CriticAbsoluteMax time.Duration
|
||||
|
||||
// SalvageSingleLoopMaxSteps opts a single-loop run into partial-transcript
|
||||
// salvage on budget exhaustion (ErrMaxSteps / ErrToolLoop / ErrMaxToolCalls):
|
||||
// instead of returning empty Output + a hard error and discarding the
|
||||
// reasoning it produced, the executor reconstructs a best-effort answer from
|
||||
// the step narration (like the phased path) and downgrades the run to a
|
||||
// successful partial result. Default false (off): a structured-output host
|
||||
// (e.g. a bounded review swarm) keeps the clean hard error, where partial
|
||||
// prose masquerading as a valid answer would be worse than a loud failure. A
|
||||
// host that wants the reasoning recovered (interactive research) sets it true.
|
||||
SalvageSingleLoopMaxSteps bool
|
||||
}
|
||||
|
||||
func (d Defaults) withFallbacks() Defaults {
|
||||
@@ -158,6 +170,7 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
if maxRuntime <= 0 {
|
||||
maxRuntime = e.cfg.Defaults.MaxRuntime
|
||||
}
|
||||
maxCalls := ra.MaxToolCalls // <= 0 => unlimited (no tool-call ceiling)
|
||||
|
||||
// Budget gate (pre-run): a rejected run makes no model call.
|
||||
if e.cfg.Ports.Budget != nil {
|
||||
@@ -202,7 +215,7 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
rec = e.cfg.Ports.Audit.StartRun(ctx, info)
|
||||
}
|
||||
if rec != nil {
|
||||
stateAcc = NewRunStateAccessor(rec, maxIter, 0, started)
|
||||
stateAcc = NewRunStateAccessor(rec, maxIter, maxCalls, started)
|
||||
inv.RunState = stateAcc
|
||||
}
|
||||
|
||||
@@ -275,6 +288,32 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
postRun = st.PostRun
|
||||
}
|
||||
|
||||
// Skill packs: resolve the agent's subscribed packs into a catalog (folded
|
||||
// into the system prompt) + a skill_use loader tool added to the toolbox.
|
||||
// nil-safe; activation failures are non-fatal — the run proceeds without
|
||||
// packs rather than dying on a fetch/cache miss.
|
||||
if len(ra.SkillPacks) > 0 && e.cfg.Ports.SkillPacks != nil {
|
||||
instr, packTools, aerr := e.cfg.Ports.SkillPacks.ActivateSkillPacks(ctx, ra.SkillPacks, inv.RunID, ra.ID)
|
||||
if aerr != nil {
|
||||
slog.Warn("run: skill-pack activation failed; continuing without packs", "run_id", inv.RunID, "error", aerr)
|
||||
} else {
|
||||
for _, t := range packTools {
|
||||
if err := toolbox.Add(t); err != nil {
|
||||
res.Err = fmt.Errorf("add skill-pack tool: %w", err)
|
||||
e.finishAudit(ctx, rec, "error", res, started, res.Err)
|
||||
return res
|
||||
}
|
||||
}
|
||||
if instr != "" {
|
||||
if ra.SystemPrompt != "" {
|
||||
ra.SystemPrompt += "\n\n" + instr
|
||||
} else {
|
||||
ra.SystemPrompt = instr
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run context: detached from the caller's deadline so a lane/queue wait doesn't
|
||||
// eat the run budget (mort's V10 lesson). Caller cancellation still propagates
|
||||
// via MergeCancellation. Created BEFORE the step observer so the observer
|
||||
@@ -345,6 +384,14 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
// calls with their executed results PAIRWISE — a result without a matching
|
||||
// call (or a call without a result) is skipped rather than recorded as an
|
||||
// empty-name "ghost" step.
|
||||
// Tool-call budget accounting (single-loop enforcement, RunnableAgent.
|
||||
// MaxToolCalls). The step observer runs on majordomo's Run goroutine — the
|
||||
// SAME goroutine that reads the step ceiling — so plain ints suffice (no
|
||||
// atomics). stepCeilingOption reads these to force the loop to exit once the
|
||||
// cap is reached; the caller then relabels the resulting ErrMaxSteps to
|
||||
// ErrMaxToolCalls. maxCalls <= 0 (unlimited) leaves this inert.
|
||||
var toolCallsSeen, stepsSeen int
|
||||
var capHit bool
|
||||
emitter := newStepEmitter(inv.OnStep)
|
||||
stepObserver := func(s agent.Step) {
|
||||
if stateAcc != nil {
|
||||
@@ -371,6 +418,21 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
rec.OnTool(call, r.Content)
|
||||
}
|
||||
}
|
||||
// Tool-call budget: accumulate executed calls and note the crossing once.
|
||||
// Only single-loop runs enforce the cap (phased pipelines are a follow-up),
|
||||
// so gate the crossing detection on Phases to avoid a misleading audit
|
||||
// event on a phased run that merely happens to carry a cap.
|
||||
toolCallsSeen += n
|
||||
stepsSeen = s.Index + 1
|
||||
if len(ra.Phases) == 0 && maxCalls > 0 && toolCallsSeen >= maxCalls && !capHit {
|
||||
capHit = true
|
||||
if rec != nil {
|
||||
rec.LogEvent("max_tool_calls_reached", map[string]any{
|
||||
"max_tool_calls": maxCalls,
|
||||
"tool_calls": toolCallsSeen,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shared agent options used by BOTH the single-loop path and every phase: the
|
||||
@@ -403,8 +465,10 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
// store and fold an [ATTACHED FILES] descriptor into the prompt so the agent
|
||||
// can reach them by file_id. No-op when Ports.InputFiles is nil or there are
|
||||
// no files. Done after the model/toolbox build but before the loop, so the
|
||||
// descriptor rides the very first user turn.
|
||||
input = e.stageInputFiles(runCtx, inv.RunID, ra.ID, inv.InputFiles, input)
|
||||
// descriptor rides the very first user turn. The staged file ids are kept for
|
||||
// the FinalGuard: they live under run scope but are the user's INPUTS, not
|
||||
// run-produced artifacts, so an undelivered-artifact check must exclude them.
|
||||
input, stagedFileIDs := e.stageInputFiles(runCtx, inv.RunID, ra.ID, inv.InputFiles, input)
|
||||
// One WithSteer drains BOTH the session mailbox (a tool's AttachImages) and
|
||||
// the critic's nudges before each step.
|
||||
steer := func() []llm.Message { return append(mailbox.drain(), critic.drainSteer()...) }
|
||||
@@ -426,6 +490,10 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
// step responses, not the loop's compacted history) — a host that caps size
|
||||
// bounds it. A recovered run seeds the saved transcript and continues.
|
||||
obs := stepObserver
|
||||
// noteCheckpoint appends an out-of-loop message (the FinalGuard nudge) to
|
||||
// the checkpoint transcript so a shutdown-resume of a nudged run replays
|
||||
// the nudge turn too. nil when the run is non-durable.
|
||||
var noteCheckpoint func(llm.Message)
|
||||
if ckpt != nil {
|
||||
var acc []llm.Message
|
||||
if resuming {
|
||||
@@ -443,10 +511,11 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
}
|
||||
_ = ckpt.Save(runCtx, RunCheckpointState{Messages: acc, Iteration: s.Index + 1})
|
||||
}
|
||||
noteCheckpoint = func(m llm.Message) { acc = append(acc, m) }
|
||||
}
|
||||
opts := append([]agent.Option{
|
||||
agent.WithToolbox(toolbox),
|
||||
critic.maxStepsOption(maxIter),
|
||||
critic.stepCeilingOption(maxIter, maxCalls, &toolCallsSeen, &stepsSeen),
|
||||
agent.WithStepObserver(obs),
|
||||
}, sharedOpts...)
|
||||
ag := agent.New(model, e.systemPrompt(ra), opts...)
|
||||
@@ -457,12 +526,130 @@ func (e *Executor) Run(ctx context.Context, ra RunnableAgent, inv tool.Invocatio
|
||||
} else {
|
||||
runRes, runErr = runAgent(runCtx, ag, input, inv.Images, agent.WithSteer(steer))
|
||||
}
|
||||
|
||||
// A tool-call-cap exit surfaces from majordomo as ErrMaxSteps (we drove
|
||||
// the step ceiling down to force the loop to stop). Relabel it so status,
|
||||
// salvage, and audit report the real cause. A capped step always carries
|
||||
// tool calls (a final-answer turn has zero), so capHit + ErrMaxSteps
|
||||
// pinpoints a cap-triggered exit.
|
||||
if capHit && errors.Is(runErr, agent.ErrMaxSteps) {
|
||||
runErr = fmt.Errorf("%w (max %d)", ErrMaxToolCalls, maxCalls)
|
||||
}
|
||||
|
||||
// ResultSchema (optional): validate the final answer and let the model
|
||||
// self-repair on the same conversation before the host sees it. Runs
|
||||
// BEFORE FinalGuard so the delivery check inspects the answer that will
|
||||
// actually be returned. Fail-open on compile problems; best-effort on
|
||||
// persistent violations (the host's fallback validator is authoritative).
|
||||
if runErr == nil && runRes != nil && len(inv.ResultSchema) > 0 {
|
||||
e.applyResultSchema(runCtx, resultSchemaDeps{
|
||||
inv: inv, ra: ra, runRes: runRes, rec: rec,
|
||||
model: model, toolbox: toolbox, sharedOpts: sharedOpts,
|
||||
obs: obs, steer: steer, noteCheckpoint: noteCheckpoint,
|
||||
})
|
||||
}
|
||||
|
||||
// FinalGuard (optional): the run finished successfully — before the result
|
||||
// is finalized, let the host inspect it for undelivered work (the
|
||||
// announce-then-stop failure: an artifact rendered into run storage, a
|
||||
// final text claiming "Done", and no delivery tool call). A non-empty
|
||||
// nudge runs ONE bounded extra round on the same conversation: same model,
|
||||
// same toolbox, same observers (audit/steps/critic all see it), but a
|
||||
// small fixed step cap so a misread nudge can't start a new work spree.
|
||||
// The nudge round's failure is non-fatal — the original successful result
|
||||
// stands (any attachments it queued before failing still deliver).
|
||||
//
|
||||
// The nudge round's job is to QUEUE deliveries (tool calls); its TEXT is
|
||||
// meta-commentary ("answer above stands", "sent!") that must never replace
|
||||
// the answer the user asked for — hosts deliver Output text + separately
|
||||
// drained attachments, so replacing Output with the nudge round's closer
|
||||
// LOSES the real answer (live regression: run d5ec39f4, a research run's
|
||||
// page-cache false positive whose nudge reply "my answer above stands
|
||||
// as-is" was delivered INSTEAD of the answer). Keep the original Output
|
||||
// unless it was blank; log what was discarded so hosts can audit.
|
||||
if runErr == nil && runRes != nil && e.cfg.Ports.FinalGuard != nil {
|
||||
state := FinalState{
|
||||
Output: runRes.Output,
|
||||
ToolNames: toolboxNames(toolbox),
|
||||
StagedFileIDs: stagedFileIDs,
|
||||
}
|
||||
if nudge := safeFinalNudge(runCtx, e.cfg.Ports.FinalGuard, info, state); nudge != "" {
|
||||
if rec != nil {
|
||||
rec.LogEvent("final_nudge", map[string]any{"nudge": nudge})
|
||||
}
|
||||
if noteCheckpoint != nil {
|
||||
noteCheckpoint(llm.UserText(nudge))
|
||||
}
|
||||
nudgeOpts := append([]agent.Option{
|
||||
agent.WithToolbox(toolbox),
|
||||
agent.WithMaxSteps(finalNudgeMaxSteps),
|
||||
agent.WithStepObserver(obs),
|
||||
}, sharedOpts...)
|
||||
nudgeAg := agent.New(model, e.systemPrompt(ra), nudgeOpts...)
|
||||
nudgeRes, nudgeErr := nudgeAg.Run(runCtx, nudge,
|
||||
agent.WithSteer(steer), agent.WithHistory(runRes.Messages))
|
||||
if nudgeRes != nil {
|
||||
// The extra round's tokens are real spend either way.
|
||||
runRes.Usage.Add(nudgeRes.Usage)
|
||||
}
|
||||
if nudgeErr != nil {
|
||||
slog.Warn("run: final-nudge round failed; keeping original result",
|
||||
"run_id", inv.RunID, "error", nudgeErr)
|
||||
} else if nudgeRes != nil {
|
||||
if strings.TrimSpace(runRes.Output) == "" {
|
||||
// Degenerate original (blank stop turn): the nudge round's
|
||||
// text is the only answer there is.
|
||||
runRes.Output = nudgeRes.Output
|
||||
} else if rec != nil {
|
||||
// Discarded meta-text stays observable: a nudge round that
|
||||
// DECLINED to queue while the original claims a send shows
|
||||
// up here, not in the user's reply.
|
||||
rec.LogEvent("final_nudge_result", map[string]any{
|
||||
"discarded_output": nudgeRes.Output,
|
||||
})
|
||||
}
|
||||
// The transcript keeps the nudge round either way (audit,
|
||||
// PostRun, checkpoint continuity).
|
||||
runRes.Messages = nudgeRes.Messages
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Single-loop budget-exhaustion salvage (opt-in). A loop that hit its
|
||||
// step/tool-call budget without composing a final answer returns empty
|
||||
// Output + a hard error, discarding the reasoning it DID produce. Mirror
|
||||
// the phased path: reconstruct a best-effort answer from the step
|
||||
// narration and downgrade the hard error to a successful partial result.
|
||||
// This runs only after a budget error — ResultSchema + FinalGuard above
|
||||
// are gated on runErr == nil, so neither fired. Gated on the host default
|
||||
// so a structured-output host keeps its clean hard error.
|
||||
if e.cfg.Defaults.SalvageSingleLoopMaxSteps &&
|
||||
isBudgetExhaustion(runErr) && runRes != nil &&
|
||||
strings.TrimSpace(runRes.Output) == "" {
|
||||
if salvaged := salvagePhaseTranscript(runRes); salvaged != "" {
|
||||
runRes.Output = salvaged +
|
||||
"\n\n(Note: reached its step budget before composing a final answer; the above is its partial reasoning.)"
|
||||
if rec != nil {
|
||||
rec.LogEvent("single_loop_salvaged", map[string]any{"error": runErr.Error()})
|
||||
}
|
||||
runErr = nil // downgrade: status becomes "ok" and Output is delivered
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Multi-phase pipeline: each phase runs its own prompt/tier/tools/step-cap
|
||||
// sequentially, threading outputs through {{.<PhaseName>}} templates. The
|
||||
// shared step observer (audit/steps/critic) is wired per phase by the phase
|
||||
// runner; checkpointing is phase-boundary granular (completed phases are
|
||||
// recorded so a resumed run skips them).
|
||||
//
|
||||
// ResultSchema is single-loop-only (FinalGuard's scope) — make the drop
|
||||
// observable rather than silent.
|
||||
if len(inv.ResultSchema) > 0 {
|
||||
slog.Warn("run: result_schema ignored on multi-phase run", "run_id", inv.RunID)
|
||||
if rec != nil {
|
||||
rec.LogEvent("result_schema_skipped", map[string]any{"error": "multi-phase runs are not schema-validated"})
|
||||
}
|
||||
}
|
||||
runRes, runErr = e.runPhases(runCtx, ra, phaseDeps{
|
||||
baseModel: model,
|
||||
baseToolbox: toolbox,
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// FinalGuard is the optional host seam consulted when a single-loop run is
|
||||
// about to finalize SUCCESSFULLY (the model produced a final answer and no run
|
||||
// error occurred). It exists for the "announce-then-stop" failure: an agent
|
||||
// renders an artifact into host storage, narrates "Done — sent you the file",
|
||||
// and ends the run without ever calling the host's delivery tool — so nothing
|
||||
// reaches the user while the final text claims otherwise (observed live:
|
||||
// gifsmith run 29170c93 2026-07-12, general run e9e7bf40 2026-07-14).
|
||||
//
|
||||
// FinalNudge returns "" to let the run finish as-is (the overwhelmingly common
|
||||
// case — the host should make its check cheap), or a non-empty nudge message.
|
||||
// A non-empty nudge sends the loop back for ONE bounded extra round: the nudge
|
||||
// is appended to the SAME conversation as the next user turn and the agent
|
||||
// runs again with the same toolbox, capped at finalNudgeMaxSteps steps — enough
|
||||
// to deliver a stranded artifact, not enough to start a new project. The guard
|
||||
// is consulted at most ONCE per run; the nudge round's own stop turn is final.
|
||||
// Multi-phase runs are not guarded (their output contract is the phase
|
||||
// pipeline's, not a delivery surface's).
|
||||
//
|
||||
// OUTPUT CONTRACT: the nudge round exists to make TOOL CALLS (queue the
|
||||
// delivery); its final TEXT is discarded — the run's Output stays the original
|
||||
// answer unless that answer was blank. A host's nudge message should say so
|
||||
// explicitly ("any text you write here is not shown to the user"), and must
|
||||
// not instruct the model to compose a corrected reply. Discarded nudge text is
|
||||
// recorded as a final_nudge_result audit event.
|
||||
//
|
||||
// The call runs on the run goroutine, inside the run's deadline, and is
|
||||
// panic-isolated: a guard panic is logged and treated as "" (the run's
|
||||
// successful result must never be lost to a guard bug).
|
||||
type FinalGuard interface {
|
||||
FinalNudge(ctx context.Context, info RunInfo, state FinalState) string
|
||||
}
|
||||
|
||||
// FinalState is the snapshot a FinalGuard receives about the finishing run.
|
||||
type FinalState struct {
|
||||
// Output is the run's final answer text (what the host would deliver).
|
||||
Output string
|
||||
// ToolNames lists the tools that were available to the run, so a guard can
|
||||
// skip nudging a run that has no way to act (e.g. no send_attachments).
|
||||
ToolNames []string
|
||||
// StagedFileIDs are the file ids the executor staged from the invocation's
|
||||
// input attachments (Ports.InputFiles). They live under the run's file scope
|
||||
// but are the USER's inputs, not run-produced artifacts — a guard checking
|
||||
// for undelivered artifacts must exclude them.
|
||||
StagedFileIDs []string
|
||||
}
|
||||
|
||||
// finalNudgeMaxSteps caps the nudge round's tool-dispatch steps. Delivering a
|
||||
// stranded artifact needs at most a file_list + a send_attachments + a final
|
||||
// text turn; the cap keeps a model that misreads the nudge from launching a
|
||||
// whole new work spree on the host's budget.
|
||||
const finalNudgeMaxSteps = 6
|
||||
|
||||
// safeFinalNudge consults the guard behind a recover so a guard panic degrades
|
||||
// to "no nudge" instead of converting a successful run into a panic-error (the
|
||||
// executor's top-level recover would otherwise stamp res.Err).
|
||||
func safeFinalNudge(ctx context.Context, g FinalGuard, info RunInfo, state FinalState) (nudge string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
slog.Error("run: FinalGuard panicked; skipping nudge",
|
||||
"run_id", info.RunID, "panic", r)
|
||||
nudge = ""
|
||||
}
|
||||
}()
|
||||
return g.FinalNudge(ctx, info, state)
|
||||
}
|
||||
|
||||
// toolboxNames flattens the run toolbox's tool names for FinalState (nil-safe).
|
||||
func toolboxNames(b *llm.Toolbox) []string {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
tools := b.Tools()
|
||||
names := make([]string, 0, len(tools))
|
||||
for _, t := range tools {
|
||||
names = append(names, t.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/tool"
|
||||
)
|
||||
|
||||
// guardFunc adapts a func to FinalGuard, counting calls and capturing the
|
||||
// snapshot it received.
|
||||
type guardFunc struct {
|
||||
nudge func(info RunInfo, state FinalState) string
|
||||
calls int
|
||||
info RunInfo
|
||||
state FinalState
|
||||
}
|
||||
|
||||
func (g *guardFunc) FinalNudge(_ context.Context, info RunInfo, state FinalState) string {
|
||||
g.calls++
|
||||
g.info = info
|
||||
g.state = state
|
||||
return g.nudge(info, state)
|
||||
}
|
||||
|
||||
// eventRecorder is captureRecorder plus LogEvent capture (for the final_nudge
|
||||
// audit event).
|
||||
type eventRecorder struct {
|
||||
captureRecorder
|
||||
events []string
|
||||
}
|
||||
|
||||
func (r *eventRecorder) LogEvent(eventType string, _ map[string]any) {
|
||||
r.events = append(r.events, eventType)
|
||||
}
|
||||
|
||||
// TestFinalGuardNudgeRunsOneExtraRound is the core announce-then-stop fix: the
|
||||
// model stops with "Done — sent!" (nothing delivered), the guard returns a
|
||||
// nudge, and the SAME conversation runs one more round — here a tool call
|
||||
// (queueing the delivery) then a closing text turn. The guard is consulted
|
||||
// exactly once (the nudge round's own stop is final), the audit log records
|
||||
// the nudge, and the ORIGINAL answer text survives: the nudge round's closer
|
||||
// is discarded (recorded as final_nudge_result), because hosts deliver Output
|
||||
// text + separately-drained attachments — replacing Output with the closer is
|
||||
// the run-d5ec39f4 lost-answer regression.
|
||||
func TestFinalGuardNudgeRunsOneExtraRound(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.Reply("Done — sent you the file!"),
|
||||
// Nudge round: an (unregistered → tolerated error result) delivery call,
|
||||
// then a meta closer that must NOT become the run output.
|
||||
fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c1", Name: "send_attachments", Arguments: []byte(`{}`)}}}),
|
||||
fake.Reply("queued the file — answer above stands"),
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
rec := &eventRecorder{}
|
||||
guard := &guardFunc{nudge: func(RunInfo, FinalState) string { return "you did not deliver the file" }}
|
||||
ex := New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: Ports{
|
||||
Audit: auditFunc{start: func(RunInfo) RunRecorder { return rec }},
|
||||
FinalGuard: guard,
|
||||
},
|
||||
})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-nudge", CallerID: "c"}, "make me a file")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "Done — sent you the file!" {
|
||||
t.Fatalf("output = %q, want the ORIGINAL answer preserved (nudge closer discarded)", res.Output)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 3 {
|
||||
t.Fatalf("model called %d times, want 3 (the nudge round must still run)", n)
|
||||
}
|
||||
if guard.calls != 1 {
|
||||
t.Fatalf("guard consulted %d times, want exactly 1 (no re-guard of the nudge round)", guard.calls)
|
||||
}
|
||||
if guard.info.RunID != "r-nudge" {
|
||||
t.Errorf("guard info.RunID = %q", guard.info.RunID)
|
||||
}
|
||||
if guard.state.Output != "Done — sent you the file!" {
|
||||
t.Errorf("guard state.Output = %q, want the pre-nudge final text", guard.state.Output)
|
||||
}
|
||||
var sawNudge, sawResult bool
|
||||
for _, ev := range rec.events {
|
||||
switch ev {
|
||||
case "final_nudge":
|
||||
sawNudge = true
|
||||
case "final_nudge_result":
|
||||
sawResult = true
|
||||
}
|
||||
}
|
||||
if !sawNudge {
|
||||
t.Errorf("audit events %v missing final_nudge", rec.events)
|
||||
}
|
||||
if !sawResult {
|
||||
t.Errorf("audit events %v missing final_nudge_result (the discarded closer must stay observable)", rec.events)
|
||||
}
|
||||
if rec.stats.Output != "Done — sent you the file!" {
|
||||
t.Errorf("audit close output = %q, want the original answer", rec.stats.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFinalGuardNudgeFillsEmptyOriginal: when the ORIGINAL stop turn was blank
|
||||
// (a degenerate run), the nudge round's text is the only answer there is — it
|
||||
// becomes the output instead of being discarded.
|
||||
func TestFinalGuardNudgeFillsEmptyOriginal(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.Reply(" \n"),
|
||||
fake.Reply("actual answer from the nudge round"),
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
guard := &guardFunc{nudge: func(RunInfo, FinalState) string { return "deliver or correct" }}
|
||||
ex := New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: Ports{FinalGuard: guard},
|
||||
})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ModelTier: "m"}, tool.Invocation{RunID: "r"}, "hi")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "actual answer from the nudge round" {
|
||||
t.Fatalf("output = %q, want the nudge round's text to fill a blank original", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFinalGuardEmptyNudgeNoExtraRound: a guard returning "" adds zero model
|
||||
// calls — the common path stays one round.
|
||||
func TestFinalGuardEmptyNudgeNoExtraRound(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("plain answer"))
|
||||
m, _ := fp.Model("m")
|
||||
guard := &guardFunc{nudge: func(RunInfo, FinalState) string { return "" }}
|
||||
ex := New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: Ports{FinalGuard: guard},
|
||||
})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ModelTier: "m"}, tool.Invocation{RunID: "r"}, "hi")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "plain answer" {
|
||||
t.Fatalf("output = %q", res.Output)
|
||||
}
|
||||
if guard.calls != 1 {
|
||||
t.Errorf("guard consulted %d times, want 1", guard.calls)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 1 {
|
||||
t.Errorf("model called %d times, want 1 (empty nudge must not re-run)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFinalGuardNudgeRoundFailureKeepsResult: the nudge round erroring (model
|
||||
// failure, deadline, step cap) must NOT damage the original successful result.
|
||||
func TestFinalGuardNudgeRoundFailureKeepsResult(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.Reply("original answer"),
|
||||
fake.Fail(errors.New("provider fell over")),
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
guard := &guardFunc{nudge: func(RunInfo, FinalState) string { return "deliver it" }}
|
||||
ex := New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: Ports{FinalGuard: guard},
|
||||
})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ModelTier: "m"}, tool.Invocation{RunID: "r"}, "hi")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("a failed nudge round must not fail the run: %v", res.Err)
|
||||
}
|
||||
if res.Output != "original answer" {
|
||||
t.Fatalf("output = %q, want the original preserved", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFinalGuardNotConsultedOnRunError: a failed run is never nudged — the
|
||||
// guard exists for successful-but-undelivered finishes only.
|
||||
func TestFinalGuardNotConsultedOnRunError(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Fail(errors.New("boom")))
|
||||
m, _ := fp.Model("m")
|
||||
guard := &guardFunc{nudge: func(RunInfo, FinalState) string { return "nudge" }}
|
||||
ex := New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: Ports{FinalGuard: guard},
|
||||
})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ModelTier: "m"}, tool.Invocation{RunID: "r"}, "hi")
|
||||
|
||||
if res.Err == nil {
|
||||
t.Fatal("expected a run error")
|
||||
}
|
||||
if guard.calls != 0 {
|
||||
t.Errorf("guard consulted %d times on a failed run, want 0", guard.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFinalGuardPanicIsolated: a panicking guard degrades to "no nudge" — the
|
||||
// successful result survives untouched.
|
||||
func TestFinalGuardPanicIsolated(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("safe answer"))
|
||||
m, _ := fp.Model("m")
|
||||
ex := New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: Ports{FinalGuard: panicGuard{}},
|
||||
})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ModelTier: "m"}, tool.Invocation{RunID: "r"}, "hi")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("guard panic must not fail the run: %v", res.Err)
|
||||
}
|
||||
if res.Output != "safe answer" {
|
||||
t.Fatalf("output = %q", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
type panicGuard struct{}
|
||||
|
||||
func (panicGuard) FinalNudge(context.Context, RunInfo, FinalState) string {
|
||||
panic("guard bug")
|
||||
}
|
||||
|
||||
// TestFinalGuardReceivesToolNamesAndStagedIDs: the guard's snapshot carries the
|
||||
// run's tool names (via ExtraTools here) and the input-staged file ids, so a
|
||||
// host can skip runs that can't deliver and exclude the user's own attachments.
|
||||
func TestFinalGuardReceivesToolNamesAndStagedIDs(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("done"))
|
||||
m, _ := fp.Model("m")
|
||||
guard := &guardFunc{nudge: func(RunInfo, FinalState) string { return "" }}
|
||||
ex := New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: Ports{
|
||||
FinalGuard: guard,
|
||||
InputFiles: guardStager(func(name string) string { return "staged-" + name }),
|
||||
},
|
||||
})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ModelTier: "m"},
|
||||
tool.Invocation{
|
||||
RunID: "r",
|
||||
ExtraTools: []llm.Tool{{
|
||||
Name: "send_attachments",
|
||||
Description: "deliver",
|
||||
Handler: func(context.Context, json.RawMessage) (any, error) { return "ok", nil },
|
||||
}},
|
||||
InputFiles: []tool.InputFile{{Name: "input.wav", MimeType: "audio/wave", Data: []byte("x")}},
|
||||
}, "hi")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
var sawTool bool
|
||||
for _, n := range guard.state.ToolNames {
|
||||
if n == "send_attachments" {
|
||||
sawTool = true
|
||||
}
|
||||
}
|
||||
if !sawTool {
|
||||
t.Errorf("guard state.ToolNames = %v, want send_attachments present", guard.state.ToolNames)
|
||||
}
|
||||
if len(guard.state.StagedFileIDs) != 1 || guard.state.StagedFileIDs[0] != "staged-input.wav" {
|
||||
t.Errorf("guard state.StagedFileIDs = %v, want [staged-input.wav]", guard.state.StagedFileIDs)
|
||||
}
|
||||
}
|
||||
|
||||
// guardStager maps an input file's name to a deterministic file id.
|
||||
type guardStager func(name string) string
|
||||
|
||||
func (f guardStager) StageInputFile(_ context.Context, _, _, name, _ string, _ []byte) (string, error) {
|
||||
return f(name), nil
|
||||
}
|
||||
|
||||
// TestFinalGuardSkipsMultiPhaseRuns: a phase pipeline's output contract is the
|
||||
// pipeline's, not a delivery surface's — the guard must not be consulted.
|
||||
func TestFinalGuardSkipsMultiPhaseRuns(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("phase-out"))
|
||||
m, _ := fp.Model("m")
|
||||
guard := &guardFunc{nudge: func(RunInfo, FinalState) string { return "nudge" }}
|
||||
ex := New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: Ports{FinalGuard: guard},
|
||||
})
|
||||
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{ModelTier: "m", Phases: []Phase{{Name: "a", SystemPrompt: "do a"}}},
|
||||
tool.Invocation{RunID: "r"}, "hi")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if guard.calls != 0 {
|
||||
t.Errorf("guard consulted %d times on a phase run, want 0", guard.calls)
|
||||
}
|
||||
}
|
||||
+11
-6
@@ -30,10 +30,11 @@ const maxInputFiles = 32
|
||||
//
|
||||
// Best-effort: a nil stager, no files, or a per-file save error degrades to
|
||||
// "skip that file" — the run still proceeds. Returns the (possibly augmented)
|
||||
// prompt.
|
||||
func (e *Executor) stageInputFiles(ctx context.Context, runID, agentID string, files []tool.InputFile, prompt string) string {
|
||||
// prompt plus the staged file ids (for the FinalGuard: staged inputs live under
|
||||
// run scope but are not run-produced artifacts).
|
||||
func (e *Executor) stageInputFiles(ctx context.Context, runID, agentID string, files []tool.InputFile, prompt string) (string, []string) {
|
||||
if e.cfg.Ports.InputFiles == nil || len(files) == 0 {
|
||||
return prompt
|
||||
return prompt, nil
|
||||
}
|
||||
// Count cap: bound how many attachments one run can stage, independent of the
|
||||
// per-file byte cap (defense-in-depth against a flood of tiny files).
|
||||
@@ -87,7 +88,11 @@ func (e *Executor) stageInputFiles(ctx context.Context, runID, agentID string, f
|
||||
staged = append(staged, stagedFile{name: name, mime: mime, fileID: sanitizeField(fileID), size: len(f.Data)})
|
||||
}
|
||||
if len(staged) == 0 {
|
||||
return prompt
|
||||
return prompt, nil
|
||||
}
|
||||
stagedIDs := make([]string, 0, len(staged))
|
||||
for _, s := range staged {
|
||||
stagedIDs = append(stagedIDs, s.fileID)
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
@@ -101,9 +106,9 @@ func (e *Executor) stageInputFiles(ctx context.Context, runID, agentID string, f
|
||||
}
|
||||
|
||||
if strings.TrimSpace(prompt) == "" {
|
||||
return b.String()
|
||||
return b.String(), stagedIDs
|
||||
}
|
||||
return prompt + "\n\n" + b.String()
|
||||
return prompt + "\n\n" + b.String(), stagedIDs
|
||||
}
|
||||
|
||||
// sanitizeName reduces an untrusted attachment filename to a safe base name. It
|
||||
|
||||
+10
-10
@@ -44,7 +44,7 @@ func newStagerExecutor(s InputFileStager) *Executor {
|
||||
func TestStageInputFiles(t *testing.T) {
|
||||
st := &stagerFunc{}
|
||||
ex := newStagerExecutor(st)
|
||||
out := ex.stageInputFiles(context.Background(), "run-1", "agent-1",
|
||||
out, _ := ex.stageInputFiles(context.Background(), "run-1", "agent-1",
|
||||
[]tool.InputFile{{Name: "clip.mp3", MimeType: "audio/mpeg", Data: []byte("abcd")}},
|
||||
"transcribe this")
|
||||
|
||||
@@ -65,7 +65,7 @@ func TestStageInputFiles(t *testing.T) {
|
||||
// drops the run.
|
||||
func TestStageInputFilesNoStager(t *testing.T) {
|
||||
ex := newStagerExecutor(nil) // Ports.InputFiles == nil
|
||||
out := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
out, _ := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
[]tool.InputFile{{Name: "x.bin", Data: []byte("z")}}, "prompt")
|
||||
if out != "prompt" {
|
||||
t.Errorf("nil stager changed the prompt: %q", out)
|
||||
@@ -75,7 +75,7 @@ func TestStageInputFilesNoStager(t *testing.T) {
|
||||
// TestStageInputFilesNoFiles: no attachments leaves the prompt untouched.
|
||||
func TestStageInputFilesNoFiles(t *testing.T) {
|
||||
ex := newStagerExecutor(&stagerFunc{})
|
||||
out := ex.stageInputFiles(context.Background(), "r", "a", nil, "prompt")
|
||||
out, _ := ex.stageInputFiles(context.Background(), "r", "a", nil, "prompt")
|
||||
if out != "prompt" {
|
||||
t.Errorf("no files changed the prompt: %q", out)
|
||||
}
|
||||
@@ -86,7 +86,7 @@ func TestStageInputFilesNoFiles(t *testing.T) {
|
||||
func TestStageInputFilesDedup(t *testing.T) {
|
||||
st := &stagerFunc{}
|
||||
ex := newStagerExecutor(st)
|
||||
out := ex.stageInputFiles(context.Background(), "r", "a", []tool.InputFile{
|
||||
out, _ := ex.stageInputFiles(context.Background(), "r", "a", []tool.InputFile{
|
||||
{Name: "a.wav", MimeType: "audio/wav", Data: []byte("1")},
|
||||
{Name: "a.wav", MimeType: "audio/wav", Data: []byte("2")},
|
||||
}, "go")
|
||||
@@ -106,13 +106,13 @@ func TestStageInputFilesDedup(t *testing.T) {
|
||||
func TestStageInputFilesSkipsBad(t *testing.T) {
|
||||
// Empty data → skipped; with no good files the prompt is returned as-is.
|
||||
ex := newStagerExecutor(&stagerFunc{})
|
||||
if out := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
if out, _ := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
[]tool.InputFile{{Name: "empty.bin", Data: nil}}, "p"); out != "p" {
|
||||
t.Errorf("empty file should be skipped, got %q", out)
|
||||
}
|
||||
// A stager error → that file is dropped; nothing staged → prompt unchanged.
|
||||
exErr := newStagerExecutor(&stagerFunc{err: errors.New("disk full")})
|
||||
if out := exErr.stageInputFiles(context.Background(), "r", "a",
|
||||
if out, _ := exErr.stageInputFiles(context.Background(), "r", "a",
|
||||
[]tool.InputFile{{Name: "x.bin", Data: []byte("z")}}, "p"); out != "p" {
|
||||
t.Errorf("save error should drop the file and leave the prompt, got %q", out)
|
||||
}
|
||||
@@ -124,7 +124,7 @@ func TestStageInputFilesOversize(t *testing.T) {
|
||||
st := &stagerFunc{}
|
||||
ex := newStagerExecutor(st)
|
||||
big := make([]byte, maxInputFileBytes+1)
|
||||
out := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
out, _ := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
[]tool.InputFile{{Name: "huge.bin", Data: big}}, "p")
|
||||
if out != "p" || len(st.staged) != 0 {
|
||||
t.Errorf("oversized file should be skipped: out=%q staged=%d", out, len(st.staged))
|
||||
@@ -177,7 +177,7 @@ func TestSanitizeName(t *testing.T) {
|
||||
func TestStageInputFilesSanitizesTraversal(t *testing.T) {
|
||||
st := &stagerFunc{}
|
||||
ex := newStagerExecutor(st)
|
||||
out := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
out, _ := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
[]tool.InputFile{{Name: "../../../etc/passwd", MimeType: "text/plain", Data: []byte("x")}}, "go")
|
||||
if len(st.staged) != 1 || st.staged[0].name != "passwd" {
|
||||
t.Fatalf("staged name = %+v, want passwd", st.staged)
|
||||
@@ -202,7 +202,7 @@ func TestSanitizeFieldStripsBidiAndControl(t *testing.T) {
|
||||
func TestStageInputFilesSanitizesMime(t *testing.T) {
|
||||
st := &stagerFunc{}
|
||||
ex := newStagerExecutor(st)
|
||||
out := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
out, _ := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
[]tool.InputFile{{Name: "c.wav", MimeType: "audio/wav\ninjected", Data: []byte("x")}}, "go")
|
||||
if len(st.staged) != 1 || strings.ContainsAny(st.staged[0].mime, "\n\r") {
|
||||
t.Errorf("mime not sanitized before staging: %+v", st.staged)
|
||||
@@ -216,7 +216,7 @@ func TestStageInputFilesSanitizesMime(t *testing.T) {
|
||||
// file (no blank file_id in the descriptor).
|
||||
func TestStageInputFilesEmptyFileID(t *testing.T) {
|
||||
ex := newStagerExecutor(emptyIDStager{})
|
||||
out := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
out, _ := ex.stageInputFiles(context.Background(), "r", "a",
|
||||
[]tool.InputFile{{Name: "x.bin", Data: []byte("z")}}, "p")
|
||||
if out != "p" {
|
||||
t.Errorf("empty file_id should drop the file, got %q", out)
|
||||
|
||||
+11
-7
@@ -156,7 +156,7 @@ func (e *Executor) runPhases(runCtx context.Context, ra RunnableAgent, deps phas
|
||||
deps.rec.LogEvent("phase_failed_optional", map[string]any{"phase": phase.Name, "error": err.Error()})
|
||||
}
|
||||
|
||||
case isPhaseBudgetExhaustion(err) && (!isLast || trimmed != ""):
|
||||
case isBudgetExhaustion(err) && (!isLast || trimmed != ""):
|
||||
// Soft stop: the phase ran out of its step/tool budget before
|
||||
// composing a final answer. Not fatal — it did real work (runOnePhase
|
||||
// salvaged its partial transcript into output), and aborting would
|
||||
@@ -248,7 +248,7 @@ func (e *Executor) runOnePhase(runCtx context.Context, ra RunnableAgent, deps ph
|
||||
}
|
||||
// Budget/guard exhaustion leaves a usable partial transcript but an empty
|
||||
// final answer; salvage the narrated work so the pipeline can carry it forward.
|
||||
if runErr != nil && isPhaseBudgetExhaustion(runErr) {
|
||||
if runErr != nil && isBudgetExhaustion(runErr) {
|
||||
if salvaged := salvagePhaseTranscript(res); salvaged != "" {
|
||||
output = salvaged
|
||||
}
|
||||
@@ -278,11 +278,15 @@ func (e *Executor) phaseModel(ctx context.Context, deps phaseDeps, ra RunnableAg
|
||||
return modelCtx, m
|
||||
}
|
||||
|
||||
// isPhaseBudgetExhaustion reports whether err is a soft budget/guard stop (the
|
||||
// loop hit its step cap or tripped a tool-error guard) — which leaves a usable
|
||||
// partial transcript — as opposed to a hard error (cancellation, model failure).
|
||||
func isPhaseBudgetExhaustion(err error) bool {
|
||||
return errors.Is(err, agent.ErrMaxSteps) || errors.Is(err, agent.ErrToolLoop)
|
||||
// isBudgetExhaustion reports whether err is a soft budget/guard stop (the loop
|
||||
// hit its step cap, its tool-call cap, or tripped a tool-error guard) — which
|
||||
// leaves a usable partial transcript — as opposed to a hard error (cancellation,
|
||||
// model failure). Shared by the phased pipeline and the single-loop salvage in
|
||||
// executor.go, so the two paths never drift on what counts as exhaustion.
|
||||
func isBudgetExhaustion(err error) bool {
|
||||
return errors.Is(err, agent.ErrMaxSteps) ||
|
||||
errors.Is(err, agent.ErrToolLoop) ||
|
||||
errors.Is(err, ErrMaxToolCalls)
|
||||
}
|
||||
|
||||
// maxSalvageBytes bounds a salvaged partial transcript so a long phase's narrated
|
||||
|
||||
@@ -16,6 +16,14 @@ import (
|
||||
// kill via KillCause(); the executor wraps that reason with this sentinel.
|
||||
var ErrCriticKill = errors.New("run: critic killed the run")
|
||||
|
||||
// ErrMaxToolCalls is the cause when a run reaches its RunnableAgent.MaxToolCalls
|
||||
// budget before composing a final answer. It surfaces from majordomo as
|
||||
// ErrMaxSteps (the kernel forces the step ceiling down to stop the loop) and is
|
||||
// relabeled to this sentinel so audit + status report the real cause. Treated
|
||||
// as budget exhaustion by isBudgetExhaustion, so the single-loop salvage path
|
||||
// recovers a partial answer from it when enabled.
|
||||
var ErrMaxToolCalls = errors.New("run: max tool calls reached without a final answer")
|
||||
|
||||
// Ports are the host seams the run executor consumes. Every field is nil-safe:
|
||||
// a light host passes the zero Ports and gets a bounded, in-memory run with no
|
||||
// persistence, audit, budget, critic, delegation, or delivery — which is
|
||||
@@ -49,6 +57,27 @@ type Ports struct {
|
||||
// are silently ignored (the run still proceeds, text-only). The bytes are
|
||||
// never inlined into the model context — the LLM can't read raw audio/binary.
|
||||
InputFiles InputFileStager
|
||||
// SkillPacks activates a RunnableAgent.SkillPacks (SKILL.md subscriptions)
|
||||
// for the run: it folds a catalog into the system prompt and adds a skill_use
|
||||
// loader tool. nil = SkillPacks are inert. The executus/skillpack battery
|
||||
// ships a default impl (skillpack.Activator).
|
||||
SkillPacks SkillPackActivator
|
||||
// FinalGuard is consulted when a single-loop run is about to finalize
|
||||
// successfully; a non-empty nudge sends the loop back for one bounded extra
|
||||
// round (the announce-then-stop undelivered-artifact check). nil = runs
|
||||
// finalize unguarded. See FinalGuard in final_guard.go.
|
||||
FinalGuard FinalGuard
|
||||
}
|
||||
|
||||
// SkillPackActivator resolves an agent's subscribed skill-pack names for a run
|
||||
// into system-prompt instructions (a catalog of what's available on demand) and
|
||||
// the tools that back them (a single skill_use loader). It receives the run +
|
||||
// subject ids so the impl can scope any per-run file staging. It returns "" +
|
||||
// nil when nothing resolves; activation errors are non-fatal to the run. Defined
|
||||
// here (the consumer) so the battery satisfies it structurally without importing
|
||||
// run — the same inversion as the other ports.
|
||||
type SkillPackActivator interface {
|
||||
ActivateSkillPacks(ctx context.Context, names []string, runID, subjectID string) (instructions string, tools []llm.Tool, err error)
|
||||
}
|
||||
|
||||
// InputFileStager persists a single non-image input attachment into a host file
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
|
||||
llm "gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/tool"
|
||||
)
|
||||
|
||||
// result_schema.go — kernel-native structured output for single-loop runs.
|
||||
//
|
||||
// When tool.Invocation.ResultSchema is set, the executor validates the loop's
|
||||
// final answer against the schema and, on violation, sends the loop back for
|
||||
// bounded corrective rounds ON THE SAME CONVERSATION — the model repairs its
|
||||
// own answer with full task context, which beats any post-hoc host-side
|
||||
// reformat (the host can't know what the model meant). Mirrors the FinalGuard
|
||||
// extra-round mechanics (same model, same observers, small step cap), with one
|
||||
// deliberate contrast: FinalGuard discards the extra round's text, while a
|
||||
// schema round's text IS the new candidate answer.
|
||||
//
|
||||
// Fail-open posture: a schema that doesn't compile logs a WARN and skips
|
||||
// validation entirely — the host's pre-dispatch compile is the authoritative
|
||||
// gate, and the kernel must not brick runs on a host bug. Persistent
|
||||
// violations keep the last candidate output; hosts layer their own fallback
|
||||
// validator/envelope on top.
|
||||
|
||||
const (
|
||||
// resultSchemaMaxBytes caps the schema the kernel will compile.
|
||||
resultSchemaMaxBytes = 64 << 10
|
||||
// resultSchemaMaxRounds caps corrective rounds per run.
|
||||
resultSchemaMaxRounds = 2
|
||||
// resultSchemaRoundMaxSteps caps each corrective round. Producing a
|
||||
// corrected JSON document needs no tools; 2 steps tolerates one stray
|
||||
// tool call before the reply.
|
||||
resultSchemaRoundMaxSteps = 2
|
||||
// resultSchemaExtractCap bounds the output size scanned for a JSON
|
||||
// document (the balanced-span scan is quadratic in the worst case).
|
||||
resultSchemaExtractCap = 64 << 10
|
||||
)
|
||||
|
||||
// denyLoader fails every $ref resolution. jsonschema/v6's default compiler
|
||||
// resolves file:// refs from local disk (and can be configured for http) —
|
||||
// a caller-supplied schema must never reach either.
|
||||
type denyLoader struct{}
|
||||
|
||||
func (denyLoader) Load(url string) (any, error) {
|
||||
return nil, fmt.Errorf("external $ref %q refused: remote/file schema loading is disabled: %w", url, fs.ErrNotExist)
|
||||
}
|
||||
|
||||
// compileResultSchema compiles a raw schema with external loading disabled.
|
||||
func compileResultSchema(raw json.RawMessage) (*jsonschema.Schema, error) {
|
||||
if len(raw) > resultSchemaMaxBytes {
|
||||
return nil, fmt.Errorf("schema is %d bytes (cap %d)", len(raw), resultSchemaMaxBytes)
|
||||
}
|
||||
doc, err := jsonschema.UnmarshalJSON(strings.NewReader(string(raw)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("schema is not valid JSON: %w", err)
|
||||
}
|
||||
c := jsonschema.NewCompiler()
|
||||
c.UseLoader(denyLoader{})
|
||||
if err := c.AddResource("inline://result-schema", doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.Compile("inline://result-schema")
|
||||
}
|
||||
|
||||
// extractResultJSON finds the answer's JSON document: the whole string, a
|
||||
// fenced block, or the first parseable balanced {...}/[...] span scanning
|
||||
// left to right. Input is capped at resultSchemaExtractCap.
|
||||
func extractResultJSON(s string) (json.RawMessage, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) > resultSchemaExtractCap {
|
||||
s = s[:resultSchemaExtractCap]
|
||||
}
|
||||
if json.Valid([]byte(s)) {
|
||||
return json.RawMessage(s), true
|
||||
}
|
||||
// Fenced block: ```json ... ``` or bare ``` ... ```.
|
||||
for _, fence := range []string{"```json", "```"} {
|
||||
if i := strings.Index(s, fence); i >= 0 {
|
||||
rest := s[i+len(fence):]
|
||||
if j := strings.Index(rest, "```"); j >= 0 {
|
||||
candidate := strings.TrimSpace(rest[:j])
|
||||
if json.Valid([]byte(candidate)) && candidate != "" {
|
||||
return json.RawMessage(candidate), true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// First parseable balanced span.
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] != '{' && s[i] != '[' {
|
||||
continue
|
||||
}
|
||||
if end, ok := balancedSpanEnd(s, i); ok {
|
||||
candidate := s[i : end+1]
|
||||
if json.Valid([]byte(candidate)) {
|
||||
return json.RawMessage(candidate), true
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// balancedSpanEnd returns the index of the bracket closing the span opened
|
||||
// at start, honoring JSON string/escape rules.
|
||||
func balancedSpanEnd(s string, start int) (int, bool) {
|
||||
depth := 0
|
||||
inStr := false
|
||||
esc := false
|
||||
for i := start; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if inStr {
|
||||
switch {
|
||||
case esc:
|
||||
esc = false
|
||||
case c == '\\':
|
||||
esc = true
|
||||
case c == '"':
|
||||
inStr = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
switch c {
|
||||
case '"':
|
||||
inStr = true
|
||||
case '{', '[':
|
||||
depth++
|
||||
case '}', ']':
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return i, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// validateResultJSON returns flattened error paths, nil when the doc conforms.
|
||||
func validateResultJSON(sch *jsonschema.Schema, doc json.RawMessage) []string {
|
||||
inst, err := jsonschema.UnmarshalJSON(strings.NewReader(string(doc)))
|
||||
if err != nil {
|
||||
return []string{"document is not valid JSON: " + err.Error()}
|
||||
}
|
||||
err = sch.Validate(inst)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
var ve *jsonschema.ValidationError
|
||||
if !errors.As(err, &ve) {
|
||||
return []string{err.Error()}
|
||||
}
|
||||
var out []string
|
||||
flattenValidationError(ve, &out)
|
||||
sort.Strings(out)
|
||||
if len(out) == 0 {
|
||||
out = []string{ve.Error()}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// flattenValidationError collects leaf causes as "/path: message" lines.
|
||||
func flattenValidationError(ve *jsonschema.ValidationError, out *[]string) {
|
||||
if len(ve.Causes) == 0 {
|
||||
loc := "/" + strings.Join(ve.InstanceLocation, "/")
|
||||
if len(ve.InstanceLocation) == 0 {
|
||||
loc = "/"
|
||||
}
|
||||
*out = append(*out, fmt.Sprintf("%s: %v", loc, ve.ErrorKind))
|
||||
return
|
||||
}
|
||||
for _, c := range ve.Causes {
|
||||
flattenValidationError(c, out)
|
||||
}
|
||||
}
|
||||
|
||||
// resultSchemaDeps bundles the per-run state applyResultSchema needs from
|
||||
// the executor's single-loop path.
|
||||
type resultSchemaDeps struct {
|
||||
inv tool.Invocation
|
||||
ra RunnableAgent
|
||||
runRes *agent.Result
|
||||
rec RunRecorder
|
||||
model llm.Model
|
||||
toolbox *llm.Toolbox
|
||||
sharedOpts []agent.Option
|
||||
obs func(agent.Step)
|
||||
steer func() []llm.Message
|
||||
noteCheckpoint func(llm.Message)
|
||||
}
|
||||
|
||||
// applyResultSchema runs the validate + corrective-round loop, mutating
|
||||
// d.runRes in place. Panic-isolated like safeFinalNudge: a bug in the
|
||||
// validator (or a pathological schema) must never convert a successful
|
||||
// run into a panic error.
|
||||
func (e *Executor) applyResultSchema(runCtx context.Context, d resultSchemaDeps) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
slog.Error("run: result_schema validation panicked; keeping run result",
|
||||
"run_id", d.inv.RunID, "panic", r)
|
||||
}
|
||||
}()
|
||||
sch, cerr := compileResultSchema(d.inv.ResultSchema)
|
||||
if cerr != nil {
|
||||
slog.Warn("run: result_schema failed to compile; skipping kernel validation",
|
||||
"run_id", d.inv.RunID, "error", cerr)
|
||||
if d.rec != nil {
|
||||
d.rec.LogEvent("result_schema_skipped", map[string]any{"error": cerr.Error()})
|
||||
}
|
||||
return
|
||||
}
|
||||
for round := 0; ; round++ {
|
||||
var errs []string
|
||||
doc, ok := extractResultJSON(d.runRes.Output)
|
||||
if ok {
|
||||
errs = validateResultJSON(sch, doc)
|
||||
if errs == nil {
|
||||
// Conforming: the bare validated document IS the answer.
|
||||
d.runRes.Output = string(doc)
|
||||
if d.rec != nil {
|
||||
d.rec.LogEvent("result_schema_valid", map[string]any{"rounds": round})
|
||||
}
|
||||
return
|
||||
}
|
||||
} else {
|
||||
errs = []string{"no JSON document found in output"}
|
||||
}
|
||||
if round >= resultSchemaMaxRounds {
|
||||
// Best effort exhausted — keep the last candidate; the host's
|
||||
// own validator decides what to do with it.
|
||||
if d.rec != nil {
|
||||
d.rec.LogEvent("result_schema_unmet", map[string]any{
|
||||
"rounds": round, "validation_errors": errs,
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
correction := resultSchemaCorrection(d.inv.ResultSchema, errs)
|
||||
if d.rec != nil {
|
||||
d.rec.LogEvent("result_schema_round", map[string]any{
|
||||
"round": round + 1, "validation_errors": errs,
|
||||
})
|
||||
}
|
||||
if d.noteCheckpoint != nil {
|
||||
d.noteCheckpoint(llm.UserText(correction))
|
||||
}
|
||||
roundOpts := append([]agent.Option{
|
||||
agent.WithToolbox(d.toolbox),
|
||||
agent.WithMaxSteps(resultSchemaRoundMaxSteps),
|
||||
agent.WithStepObserver(d.obs),
|
||||
}, d.sharedOpts...)
|
||||
roundAg := agent.New(d.model, e.systemPrompt(d.ra), roundOpts...)
|
||||
roundRes, roundErr := roundAg.Run(runCtx, correction,
|
||||
agent.WithSteer(d.steer), agent.WithHistory(d.runRes.Messages))
|
||||
if roundRes != nil {
|
||||
roundUsage := roundRes.Usage
|
||||
d.runRes.Usage.Add(roundUsage)
|
||||
}
|
||||
if roundErr != nil || roundRes == nil {
|
||||
slog.Warn("run: result_schema corrective round failed; keeping last output",
|
||||
"run_id", d.inv.RunID, "error", roundErr)
|
||||
return
|
||||
}
|
||||
// The round's text is the NEW candidate (contrast FinalGuard, whose
|
||||
// extra-round text is discarded meta-commentary) — but a BLANK round
|
||||
// reply must not clobber the prior non-empty candidate.
|
||||
if strings.TrimSpace(roundRes.Output) != "" {
|
||||
d.runRes.Output = roundRes.Output
|
||||
}
|
||||
d.runRes.Messages = roundRes.Messages
|
||||
}
|
||||
}
|
||||
|
||||
// resultSchemaCorrection builds the corrective user turn for one round.
|
||||
func resultSchemaCorrection(raw json.RawMessage, errs []string) string {
|
||||
return fmt.Sprintf(`Your final answer must be a SINGLE JSON document conforming to this JSON Schema (no prose, no code fences):
|
||||
%s
|
||||
|
||||
Your previous answer failed validation:
|
||||
- %s
|
||||
|
||||
Reply now with ONLY the corrected JSON document. Preserve your answer's content; do not invent data; use null or empty values for anything missing.`,
|
||||
string(raw), strings.Join(errs, "\n- "))
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
llm "gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/tool"
|
||||
)
|
||||
|
||||
func resultSchemaExecutor(t *testing.T, m llm.Model, rec RunRecorder) *Executor {
|
||||
t.Helper()
|
||||
ports := Ports{}
|
||||
if rec != nil {
|
||||
ports.Audit = auditFunc{start: func(RunInfo) RunRecorder { return rec }}
|
||||
}
|
||||
return New(Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil },
|
||||
Ports: ports,
|
||||
})
|
||||
}
|
||||
|
||||
var personSchema = json.RawMessage(`{"type":"object","required":["name"],"properties":{"name":{"type":"string"}},"additionalProperties":false}`)
|
||||
|
||||
// TestResultSchema_ConformingAnswerPassesThrough — a first-try conforming
|
||||
// answer (fence-wrapped, even) is replaced by the bare JSON with no extra
|
||||
// model rounds.
|
||||
func TestResultSchema_ConformingAnswerPassesThrough(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("Here you go:\n```json\n{\"name\": \"mort\"}\n```"))
|
||||
m, _ := fp.Model("m")
|
||||
|
||||
ex := resultSchemaExecutor(t, m, nil)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs1", CallerID: "c", ResultSchema: personSchema}, "who are you")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != `{"name": "mort"}` {
|
||||
t.Fatalf("output = %q, want the bare validated JSON", res.Output)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 1 {
|
||||
t.Fatalf("model calls = %d, want 1 (no corrective round)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResultSchema_CorrectiveRoundRepairs — a non-conforming first answer
|
||||
// triggers one corrective round whose text becomes the run output.
|
||||
func TestResultSchema_CorrectiveRoundRepairs(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.Reply(`{"name": 42}`), // wrong type
|
||||
fake.Reply(`{"name": "fixed"}`), // corrective round conforms
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
rec := &eventRecorder{}
|
||||
|
||||
ex := resultSchemaExecutor(t, m, rec)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs2", CallerID: "c", ResultSchema: personSchema}, "go")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != `{"name": "fixed"}` {
|
||||
t.Fatalf("output = %q, want the corrected JSON", res.Output)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 2 {
|
||||
t.Fatalf("model calls = %d, want 2", n)
|
||||
}
|
||||
if !recHas(rec, "result_schema_round") || !recHas(rec, "result_schema_valid") {
|
||||
t.Fatalf("expected round + valid audit events, got %v", rec.events)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResultSchema_PersistentViolationKeepsLastOutput — rounds exhaust, the
|
||||
// last candidate stands, and result_schema_unmet is recorded (the host's own
|
||||
// validator is the fallback authority).
|
||||
func TestResultSchema_PersistentViolationKeepsLastOutput(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.Reply("not json at all"),
|
||||
fake.Reply("still prose"),
|
||||
fake.Reply("prose forever"),
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
rec := &eventRecorder{}
|
||||
|
||||
ex := resultSchemaExecutor(t, m, rec)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs3", CallerID: "c", ResultSchema: personSchema}, "go")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "prose forever" {
|
||||
t.Fatalf("output = %q, want the last candidate kept", res.Output)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 1+resultSchemaMaxRounds {
|
||||
t.Fatalf("model calls = %d, want %d", n, 1+resultSchemaMaxRounds)
|
||||
}
|
||||
if !recHas(rec, "result_schema_unmet") {
|
||||
t.Fatalf("expected result_schema_unmet, got %v", rec.events)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResultSchema_NoSchemaUnchanged — anchor: without a schema the output
|
||||
// and call count are untouched.
|
||||
func TestResultSchema_NoSchemaUnchanged(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("plain prose answer"))
|
||||
m, _ := fp.Model("m")
|
||||
|
||||
ex := resultSchemaExecutor(t, m, nil)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs4", CallerID: "c"}, "go")
|
||||
|
||||
if res.Err != nil || res.Output != "plain prose answer" {
|
||||
t.Fatalf("output = %q err = %v", res.Output, res.Err)
|
||||
}
|
||||
if n := fp.CallCount("m"); n != 1 {
|
||||
t.Fatalf("model calls = %d, want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResultSchema_CompileFailureFailsOpen — an uncompilable schema skips
|
||||
// validation (WARN + audit event), never bricks the run.
|
||||
func TestResultSchema_CompileFailureFailsOpen(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m", fake.Reply("prose"))
|
||||
m, _ := fp.Model("m")
|
||||
rec := &eventRecorder{}
|
||||
|
||||
ex := resultSchemaExecutor(t, m, rec)
|
||||
res := ex.Run(context.Background(),
|
||||
RunnableAgent{Name: "g", ModelTier: "m"},
|
||||
tool.Invocation{RunID: "r-rs5", CallerID: "c",
|
||||
ResultSchema: json.RawMessage(`{"type": "not-a-type"}`)}, "go")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("run error: %v", res.Err)
|
||||
}
|
||||
if res.Output != "prose" {
|
||||
t.Fatalf("output = %q, want untouched", res.Output)
|
||||
}
|
||||
if !recHas(rec, "result_schema_skipped") {
|
||||
t.Fatalf("expected result_schema_skipped, got %v", rec.events)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompileResultSchema_ExternalRefsFailClosed — http(s) AND file $refs
|
||||
// must not resolve (jsonschema/v6's default loader reads local disk).
|
||||
func TestCompileResultSchema_ExternalRefsFailClosed(t *testing.T) {
|
||||
for _, ref := range []string{
|
||||
"https://example.com/schema.json",
|
||||
"file:///etc/passwd",
|
||||
} {
|
||||
raw := json.RawMessage(`{"$ref": "` + ref + `"}`)
|
||||
if _, err := compileResultSchema(raw); err == nil {
|
||||
t.Errorf("$ref %q compiled — external loading must fail closed", ref)
|
||||
} else if !strings.Contains(err.Error(), "refused") && !strings.Contains(err.Error(), "not exist") {
|
||||
// Any error is acceptable as long as it FAILS; the refused
|
||||
// wording just confirms the deny loader was consulted.
|
||||
t.Logf("$ref %q failed with: %v", ref, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractResultJSON_Shapes — extraction tolerance.
|
||||
func TestExtractResultJSON_Shapes(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
{`{"a":1}`, `{"a":1}`, true},
|
||||
{"prefix text {\"a\": {\"b\": \"}\"}} suffix", `{"a": {"b": "}"}}`, true},
|
||||
{"```json\n[1,2]\n```", `[1,2]`, true},
|
||||
{"no json here", "", false},
|
||||
{"broken { \"a\": ", "", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, ok := extractResultJSON(c.in)
|
||||
if ok != c.ok || (ok && string(got) != c.want) {
|
||||
t.Errorf("extract(%q) = (%q, %v), want (%q, %v)", c.in, got, ok, c.want, c.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recHas reports whether the recorder saw an event type.
|
||||
func recHas(r *eventRecorder, name string) bool {
|
||||
for _, e := range r.events {
|
||||
if e == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package run_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/agent"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/provider/fake"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/run"
|
||||
"gitea.stevedudenhoeffer.com/steve/executus/tool"
|
||||
)
|
||||
|
||||
// salvageNoopTool is an always-succeeds tool injected via Invocation.ExtraTools
|
||||
// so a scripted tool-call loop advances (and counts a real tool call) without
|
||||
// tripping the tool-error guard.
|
||||
func salvageNoopTool() []llm.Tool {
|
||||
return []llm.Tool{{
|
||||
Name: "noop",
|
||||
Description: "no-op",
|
||||
Handler: func(context.Context, json.RawMessage) (any, error) { return "ok", nil },
|
||||
}}
|
||||
}
|
||||
|
||||
// narratingLoop returns a fake provider whose every reply NARRATES a step and
|
||||
// calls noop with DISTINCT args (so the same-call-repeat guard never trips), so
|
||||
// the loop never finalizes and runs until its step / tool-call budget. Each
|
||||
// step's narration is recoverable by the salvage path.
|
||||
func narratingLoop() *fake.Provider {
|
||||
var mu sync.Mutex
|
||||
var n int
|
||||
return fake.New("fake", fake.WithDefault(func(_ string, _ llm.Request) fake.Step {
|
||||
mu.Lock()
|
||||
n++
|
||||
step := n
|
||||
mu.Unlock()
|
||||
return fake.ReplyWith(llm.Response{
|
||||
Parts: []llm.Part{llm.Text(fmt.Sprintf("reasoning %d: partial finding %d", step, step))},
|
||||
ToolCalls: []llm.ToolCall{{
|
||||
ID: fmt.Sprintf("c%d", step),
|
||||
Name: "noop",
|
||||
Arguments: []byte(fmt.Sprintf(`{"n":%d}`, step)),
|
||||
}},
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
// silentLoop is like narratingLoop but writes NO prose — so there is nothing for
|
||||
// the salvage path to recover.
|
||||
func silentLoop() *fake.Provider {
|
||||
var mu sync.Mutex
|
||||
var n int
|
||||
return fake.New("fake", fake.WithDefault(func(_ string, _ llm.Request) fake.Step {
|
||||
mu.Lock()
|
||||
n++
|
||||
step := n
|
||||
mu.Unlock()
|
||||
return fake.ReplyWith(llm.Response{
|
||||
ToolCalls: []llm.ToolCall{{
|
||||
ID: fmt.Sprintf("c%d", step),
|
||||
Name: "noop",
|
||||
Arguments: []byte(fmt.Sprintf(`{"n":%d}`, step)),
|
||||
}},
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
func modelsFor(m llm.Model) run.ModelResolver {
|
||||
return func(ctx context.Context, _ string) (context.Context, llm.Model, error) { return ctx, m, nil }
|
||||
}
|
||||
|
||||
// TestSalvageSingleLoop_OnRecoversPartialReasoning: with the salvage default ON,
|
||||
// a single-loop run that exhausts its step budget without a final answer is
|
||||
// downgraded to a successful PARTIAL result carrying its step narration, instead
|
||||
// of returning empty output + a hard ErrMaxSteps.
|
||||
func TestSalvageSingleLoop_OnRecoversPartialReasoning(t *testing.T) {
|
||||
fp := narratingLoop()
|
||||
m, _ := fp.Model("m")
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: modelsFor(m),
|
||||
Defaults: run.Defaults{SalvageSingleLoopMaxSteps: true},
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "researcher", ModelTier: "m", MaxIterations: 3},
|
||||
tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
|
||||
"do research")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("salvage ON: budget exhaustion should downgrade to success; got err %v", res.Err)
|
||||
}
|
||||
if !strings.Contains(res.Output, "partial finding") {
|
||||
t.Errorf("salvaged output should contain the step narration; got %q", res.Output)
|
||||
}
|
||||
if !strings.Contains(res.Output, "step budget") {
|
||||
t.Errorf("salvaged output should carry the partial-answer note; got %q", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSalvageSingleLoop_OffKeepsHardError: with salvage OFF (the zero-value
|
||||
// default), the same run returns empty output + a hard ErrMaxSteps — the
|
||||
// structured-output host's clean failure.
|
||||
func TestSalvageSingleLoop_OffKeepsHardError(t *testing.T) {
|
||||
fp := narratingLoop()
|
||||
m, _ := fp.Model("m")
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: modelsFor(m),
|
||||
// Defaults zero => SalvageSingleLoopMaxSteps false.
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 3},
|
||||
tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
|
||||
"go")
|
||||
|
||||
if !errors.Is(res.Err, agent.ErrMaxSteps) {
|
||||
t.Fatalf("salvage OFF: expected ErrMaxSteps, got %v", res.Err)
|
||||
}
|
||||
if strings.TrimSpace(res.Output) != "" {
|
||||
t.Errorf("salvage OFF: output should be empty; got %q", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSalvageSingleLoop_NoProseStillErrors: salvage recovers nothing when the
|
||||
// loop wrote no prose, so a hard error stands even with salvage ON.
|
||||
func TestSalvageSingleLoop_NoProseStillErrors(t *testing.T) {
|
||||
fp := silentLoop()
|
||||
m, _ := fp.Model("m")
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: modelsFor(m),
|
||||
Defaults: run.Defaults{SalvageSingleLoopMaxSteps: true},
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 3},
|
||||
tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
|
||||
"go")
|
||||
|
||||
if !errors.Is(res.Err, agent.ErrMaxSteps) {
|
||||
t.Fatalf("no prose to salvage: ErrMaxSteps should stand, got %v", res.Err)
|
||||
}
|
||||
if strings.TrimSpace(res.Output) != "" {
|
||||
t.Errorf("output should be empty; got %q", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMaxToolCalls_ForcesExitAndSurfacesSentinel: a run capped at 2 tool calls
|
||||
// stops after the 2nd (well before its high step ceiling) and surfaces the
|
||||
// distinct ErrMaxToolCalls sentinel.
|
||||
func TestMaxToolCalls_ForcesExitAndSurfacesSentinel(t *testing.T) {
|
||||
fp := narratingLoop()
|
||||
m, _ := fp.Model("m")
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: modelsFor(m),
|
||||
// salvage OFF so the raw sentinel is observable.
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 50, MaxToolCalls: 2},
|
||||
tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
|
||||
"go")
|
||||
|
||||
if !errors.Is(res.Err, run.ErrMaxToolCalls) {
|
||||
t.Fatalf("expected ErrMaxToolCalls, got %v", res.Err)
|
||||
}
|
||||
// It must NOT read as a plain step-budget exhaustion.
|
||||
if !strings.Contains(res.Err.Error(), "max tool calls") {
|
||||
t.Errorf("error should name the tool-call cap; got %q", res.Err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// TestMaxToolCalls_SalvagedWhenEnabled: a tool-call-cap exit is treated as budget
|
||||
// exhaustion, so with salvage ON it recovers the partial reasoning too.
|
||||
func TestMaxToolCalls_SalvagedWhenEnabled(t *testing.T) {
|
||||
fp := narratingLoop()
|
||||
m, _ := fp.Model("m")
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: modelsFor(m),
|
||||
Defaults: run.Defaults{SalvageSingleLoopMaxSteps: true},
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 50, MaxToolCalls: 2},
|
||||
tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
|
||||
"go")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("cap + salvage: expected downgraded success, got %v", res.Err)
|
||||
}
|
||||
if !strings.Contains(res.Output, "partial finding") {
|
||||
t.Errorf("expected salvaged narration, got %q", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMaxToolCalls_ZeroIsUnbounded: MaxToolCalls <= 0 imposes no ceiling — a run
|
||||
// that makes several tool calls then finalizes completes normally (regression:
|
||||
// unlimited must be a true no-op).
|
||||
func TestMaxToolCalls_ZeroIsUnbounded(t *testing.T) {
|
||||
fp := fake.New("fake")
|
||||
fp.Enqueue("m",
|
||||
fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c1", Name: "noop", Arguments: []byte(`{"n":1}`)}}}),
|
||||
fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c2", Name: "noop", Arguments: []byte(`{"n":2}`)}}}),
|
||||
fake.ReplyWith(llm.Response{ToolCalls: []llm.ToolCall{{ID: "c3", Name: "noop", Arguments: []byte(`{"n":3}`)}}}),
|
||||
fake.Reply("final answer after 3 tools"),
|
||||
)
|
||||
m, _ := fp.Model("m")
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: modelsFor(m),
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 10, MaxToolCalls: 0},
|
||||
tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
|
||||
"go")
|
||||
|
||||
if res.Err != nil {
|
||||
t.Fatalf("unbounded run should complete: %v", res.Err)
|
||||
}
|
||||
if res.Output != "final answer after 3 tools" {
|
||||
t.Errorf("output = %q, want the final answer", res.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMaxToolCalls_CapOverridesCriticRaise: the tool-call cap is a HARD budget —
|
||||
// it stops the run even when a critic is willing to raise the step ceiling far
|
||||
// past it. Also proves stepCeilingOption composes with a critic.
|
||||
func TestMaxToolCalls_CapOverridesCriticRaise(t *testing.T) {
|
||||
h := &fakeCriticHandle{maxSteps: 50}
|
||||
fp := narratingLoop()
|
||||
m, _ := fp.Model("m")
|
||||
ex := run.New(run.Config{
|
||||
Registry: tool.NewRegistry(),
|
||||
Models: modelsFor(m),
|
||||
Ports: run.Ports{Critic: &fakeCritic{h: h}},
|
||||
})
|
||||
res := ex.Run(context.Background(),
|
||||
run.RunnableAgent{Name: "x", ModelTier: "m", MaxIterations: 1, MaxToolCalls: 2, Critic: run.CriticConfig{Enabled: true}},
|
||||
tool.Invocation{RunID: "r", ExtraTools: salvageNoopTool()},
|
||||
"go")
|
||||
|
||||
if !errors.Is(res.Err, run.ErrMaxToolCalls) {
|
||||
t.Fatalf("the tool-call cap must override the critic's raised ceiling; got %v", res.Err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package skillpack
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// Activator adapts the battery to executus/run's SkillPackActivator port: given
|
||||
// an agent's subscribed pack names, it resolves them to their pinned packs and
|
||||
// returns the catalog instructions + the skill_use tool the run injects. It
|
||||
// satisfies run.SkillPackActivator structurally — no import of run — so the
|
||||
// battery stays run-agnostic (the same inversion as the other batteries).
|
||||
//
|
||||
// StagerFor, when set, builds the per-run BundleStager (a host plumbs bundled
|
||||
// files into its own run-scoped storage from the run + subject ids); nil means
|
||||
// skill_use lists a pack's bundled filenames without staging them.
|
||||
type Activator struct {
|
||||
Cache PackCache
|
||||
Subs Store
|
||||
StagerFor func(runID, subjectID string) BundleStager
|
||||
}
|
||||
|
||||
// ActivateSkillPacks implements run.SkillPackActivator. Unknown or disabled pack
|
||||
// names are skipped; it returns "" + nil when nothing resolves.
|
||||
func (a *Activator) ActivateSkillPacks(ctx context.Context, names []string, runID, subjectID string) (string, []llm.Tool, error) {
|
||||
if a == nil || a.Subs == nil || a.Cache == nil || len(names) == 0 {
|
||||
return "", nil, nil
|
||||
}
|
||||
chosen := make([]Subscription, 0, len(names))
|
||||
for _, n := range names {
|
||||
sub, err := a.Subs.GetByName(ctx, n)
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if !sub.Enabled {
|
||||
continue
|
||||
}
|
||||
chosen = append(chosen, *sub)
|
||||
}
|
||||
packs, err := Resolve(ctx, a.Cache, chosen)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var stager BundleStager
|
||||
if a.StagerFor != nil {
|
||||
stager = a.StagerFor(runID, subjectID)
|
||||
}
|
||||
sk := Activate(packs, stager)
|
||||
if sk == nil {
|
||||
return "", nil, nil
|
||||
}
|
||||
return sk.Instructions(), sk.Tools().Tools(), nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package skillpack
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestActivator(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
src := &fakeSource{tree: packTree("alpha", "do alpha things"), ref: "r1"}
|
||||
y := newTestSyncer(src)
|
||||
if _, err := y.Subscribe(ctx, src, "main", "steve"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
staged := 0
|
||||
act := &Activator{
|
||||
Cache: y.Cache, Subs: y.Subs,
|
||||
StagerFor: func(runID, subjectID string) BundleStager {
|
||||
return func(context.Context, *Pack) (string, error) { staged++; return "", nil }
|
||||
},
|
||||
}
|
||||
|
||||
instr, tools, err := act.ActivateSkillPacks(ctx, []string{"alpha"}, "run1", "agent1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if instr == "" {
|
||||
t.Error("expected catalog instructions")
|
||||
}
|
||||
found := false
|
||||
for _, tl := range tools {
|
||||
if tl.Name == "skill_use" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("expected a skill_use tool, got %d tools", len(tools))
|
||||
}
|
||||
|
||||
// unknown name → nothing resolves (no error, no tools).
|
||||
if in, tl, err := act.ActivateSkillPacks(ctx, []string{"nope"}, "r", "a"); err != nil || in != "" || tl != nil {
|
||||
t.Fatalf("unknown pack should resolve to nothing: in=%q tools=%v err=%v", in, tl, err)
|
||||
}
|
||||
|
||||
// nil-safe: a zero Activator (or empty names) is inert.
|
||||
if in, tl, err := (&Activator{}).ActivateSkillPacks(ctx, []string{"alpha"}, "r", "a"); err != nil || in != "" || tl != nil {
|
||||
t.Fatalf("zero Activator should be inert: %q %v %v", in, tl, err)
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ package tool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -238,6 +239,20 @@ type Invocation struct {
|
||||
// persisted agent row. Skill runs ignore this field.
|
||||
SystemPromptPrepend string
|
||||
|
||||
// ResultSchema, when non-empty, is a JSON Schema (draft 2020-12) the
|
||||
// run's final answer must conform to. The executor validates the
|
||||
// single-loop final output against it and, on violation, runs up to
|
||||
// two bounded corrective rounds on the same conversation so the
|
||||
// model self-repairs with full task context; a conforming answer is
|
||||
// replaced by the bare validated JSON document. The kernel is
|
||||
// FAIL-OPEN on schema problems (an uncompilable schema logs a WARN
|
||||
// and skips validation — the host's own pre-dispatch compile is the
|
||||
// authoritative gate) and best-effort on persistent violations (the
|
||||
// last output stands; hosts keep their own fallback validator /
|
||||
// error envelope). Multi-phase runs are not validated, matching
|
||||
// FinalGuard's single-loop-only scope.
|
||||
ResultSchema json.RawMessage
|
||||
|
||||
// SuppressDelivery, when true, instructs the skill executor to
|
||||
// SKIP its OutputTarget Delivery (Deliver / DeliverError) entirely.
|
||||
// The run still produces an output string (returned from Run) and
|
||||
|
||||
Reference in New Issue
Block a user