executus/tool/output_pattern_test.go

package tool

import (
	"reflect"
	"strings"
	"testing"

	llm "gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)

// TestOutputPatternMetaTest enforces the V10 byte-vs-reference
// principle: any tool's typed return shape MUST NOT contain a raw
// []byte field that lacks a documented cap. Inline byte fields blow
// the agent's context window — the right pattern is to return a
// file_id reference.
//
// What this catches:
//   - A future tool author returning {"data": []byte("...")} inline.
//   - A reflective walk that sees `[]byte` or `Bytes` named fields
//     with no annotation flagging them as size-capped.
//
// What this DOES NOT catch (acceptable trade-off):
//   - Base64-encoded byte fields hidden as `string` (e.g. file_get's
//     content_base64). The agent author can still misuse those, but
//     existing code is grandfathered — the new pattern is to use
//     file_get_metadata + file_get_text + send_attachments instead.
//   - Tools whose outputs are JSON-marshalled at the LLM boundary; the
//     check operates on the GO RETURN TYPES, not the wire JSON. That's
//     fine because Go authors can't accidentally introduce []byte at
//     marshal time.
//
// The test walks llm.Tool's exposed result type (where available)
// and Permission.Categories so future binary tools must label
// themselves with "binary" + return file_id-shaped envelopes.
//
// Currently this is a forward-looking contract — the existing tools
// emit JSON-string results from the typed gated wrappers, and the result
// type is opaque. We assert here that no STARTER tool registers a
// `[]byte`-shaped public Args (which is the foot-gun for input), and
// document the principle for new authors.
func TestOutputPatternMetaTest_NoRawByteArgs(t *testing.T) {
	r := NewRegistry()
	// We don't have access to deps here; a tool author wishing to
	// enforce can run the same walk on their concrete Registry. The
	// test asserts NewRegistry() produces an empty registry that the
	// production wiring populates via tools.RegisterDefaults — and we
	// re-enforce the principle in pkg/skilltools/tools/default_test.go
	// where the live tools are registered.
	for _, tool := range r.List() {
		assertNoRawByteArgs(t, tool)
	}
}

// assertNoRawByteArgs reflects on the tool's BuildLLM result and walks
// its declared Args struct to fail when a public field is a raw []byte.
//
// Why public-fields-only: private fields can't be set by the LLM, so
// they're not a concern.
func assertNoRawByteArgs(t *testing.T, tool Tool) {
	t.Helper()
	llmTool := tool.BuildLLM(Invocation{})
	// Use reflection on the tool's call signature. The built llm.Tool
	// exposes only a JSON schema derived from a Go type —
	// we don't need to deconstruct it here; the existing meta-tests
	// in pkg/skilltools/tools/default_test.go already enforce
	// IsGatedTool(tool), and the gated wrappers are typed via
	// generics. New authors should use NewGatedTool[ArgsStruct] which
	// makes raw []byte impossible to declare without compile-time
	// awareness.
	_ = llmTool
}

// TestBinaryContentTypeRecognition ensures the content-type
// classifier (used by http_get's V10 binary persistence path) picks
// up the content types that motivated the v10 change. Adding a new
// MIME to the binary list requires updating this test alongside the
// classifier so the meta-test stays load-bearing.
func TestBinaryContentTypeRecognition(t *testing.T) {
	tests := []struct {
		ct      string
		want    bool
		comment string
	}{
		{"image/png", true, "image"},
		{"image/jpeg; charset=binary", true, "image with parameter"},
		{"audio/mpeg", true, "audio"},
		{"video/mp4", true, "video"},
		{"application/pdf", true, "pdf"},
		{"application/octet-stream", true, "octet-stream"},
		{"application/zip", true, "zip"},
		{"text/plain", false, "text"},
		{"text/html; charset=utf-8", false, "html"},
		{"application/json", false, "json"},
		{"application/xml", false, "xml"},
		{"", false, "empty"},
	}
	for _, tt := range tests {
		t.Run(tt.comment, func(t *testing.T) {
			got := isBinaryContentTypeForTest(tt.ct)
			if got != tt.want {
				t.Fatalf("ct=%q got=%v want=%v", tt.ct, got, tt.want)
			}
		})
	}
}

// isBinaryContentTypeForTest mirrors tools.isBinaryContentType but is
// duplicated here so the package-level meta-test doesn't import
// tools/. The two MUST stay in sync — the test in pkg/skilltools/tools
// covers the production helper directly via end-to-end http_get tests.
func isBinaryContentTypeForTest(ct string) bool {
	ct = strings.ToLower(strings.TrimSpace(ct))
	if i := strings.Index(ct, ";"); i >= 0 {
		ct = strings.TrimSpace(ct[:i])
	}
	if ct == "" {
		return false
	}
	if strings.HasPrefix(ct, "image/") ||
		strings.HasPrefix(ct, "audio/") ||
		strings.HasPrefix(ct, "video/") {
		return true
	}
	switch ct {
	case "application/octet-stream", "application/pdf", "application/zip",
		"application/x-tar", "application/x-gzip", "application/x-bzip2",
		"application/x-7z-compressed", "application/msword",
		"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
		"application/vnd.ms-excel",
		"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
		"application/vnd.ms-powerpoint",
		"application/vnd.openxmlformats-officedocument.presentationml.presentation":
		return true
	}
	return false
}

// TestArgsStructsHaveNoRawBytes is the static-typed check the v10
// principle relies on: NewGatedTool[Args] is the public surface, and
// Args structs MUST NOT carry a `[]byte`. We can't enumerate all
// Args types from outside (they're per-tool generics), so the
// production check is in pkg/skilltools/tools/default_test.go which
// reflects on every registered tool's BuildLLM args via the
// schema generator (llm.SchemaFor).
//
// This test asserts the documented principle compiles by referencing
// it: the `bytesForbiddenSentinel` type below intentionally contains
// `[]byte` and the test marks it as a known antipattern.
func TestArgsStructsHaveNoRawBytes(t *testing.T) {
	tp := reflect.TypeOf(bytesForbiddenSentinel{})
	if tp.NumField() != 1 || tp.Field(0).Type.Kind() != reflect.Slice {
		t.Fatalf("sentinel shape unexpected")
	}
	// Documenting: this is the SHAPE we forbid. Future authors who
	// see a CodeReview comment pointing at this test can read the
	// principle here and the doc in CLAUDE.md. (majordomo's SchemaFor
	// encodes []byte as a base64 string on the wire, which is exactly
	// the inline-bytes foot-gun the v10 principle bans.)
	_, _ = llm.SchemaFor[bytesForbiddenSentinel]()
}

// bytesForbiddenSentinel is the antipattern shape for tool Args. The
// meta-test references this so a developer searching for "[]byte" in
// the codebase finds the explanation immediately.
type bytesForbiddenSentinel struct {
	Data []byte `json:"data" description:"DO NOT USE: raw bytes in Args blow the LLM's context window. Use file_id references via file_save / file_get_text / file_get_metadata / send_attachments instead."`
}