From 434d721b996c54ecdf23c59234f166d095a9d134 Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Sat, 11 Jul 2026 23:24:14 -0400 Subject: [PATCH 1/2] feat: audio surfaces (TTS + transcription), imagegen.Editor, llamaswap health probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New leaf package `audio` (ADR-0017): SpeechModel/SpeechProvider and TranscriptionModel/TranscriptionProvider with imagegen conventions (zero value = backend default, functional options + Apply, bytes in/out, never URLs). Root re-exports added. - imagegen.Editor (ADR-0018): optional image-to-image interface — EditRequest carries the generation knobs plus Init image and denoising Strength; separate interface so existing Models keep compiling. - provider/llamaswap implements all of it: POST /v1/audio/speech (JSON, raw-audio response, MIME from Content-Type with format fallback), POST /v1/audio/transcriptions (multipart, response_format=json), ListVoices (GET /v1/audio/voices?model=, tolerant of string-list and object-list shapes), POST /sdapi/v1/img2img (txt2img wire + init_images/denoising_strength, shared image decode), and Health(ctx) (GET /health) — a cheap liveness probe for often-offline hosts. - Hermetic httptest coverage for every new wire shape and validation path; README sections + support-matrix footnote updated in the same commit (also corrects the stale /v1/images/generations claim — the image path has been SDAPI since the seed fix). First consumer: mort's llamaswap media tool cluster (status / image / TTS / STT agent tools against the netherstorm host). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AXQxVhXBw8PwFAtsVrXSmj --- README.md | 50 +++++- audio/audio.go | 186 ++++++++++++++++++++ audio/audio_test.go | 38 ++++ docs/adr/0017-audio-interfaces.md | 53 ++++++ docs/adr/0018-imagegen-editor.md | 36 ++++ docs/adr/README.md | 2 + imagegen/edit.go | 91 ++++++++++ imagegen/edit_test.go | 26 +++ majordomo.go | 21 +++ progress.md | 16 ++ provider/llamaswap/audio.go | 278 ++++++++++++++++++++++++++++++ provider/llamaswap/audio_test.go | 247 ++++++++++++++++++++++++++ provider/llamaswap/edit_test.go | 110 ++++++++++++ provider/llamaswap/health.go | 45 +++++ provider/llamaswap/health_test.go | 59 +++++++ provider/llamaswap/image.go | 67 ++++++- 16 files changed, 1317 insertions(+), 8 deletions(-) create mode 100644 audio/audio.go create mode 100644 audio/audio_test.go create mode 100644 docs/adr/0017-audio-interfaces.md create mode 100644 docs/adr/0018-imagegen-editor.md create mode 100644 imagegen/edit.go create mode 100644 imagegen/edit_test.go create mode 100644 provider/llamaswap/audio.go create mode 100644 provider/llamaswap/audio_test.go create mode 100644 provider/llamaswap/edit_test.go create mode 100644 provider/llamaswap/health.go create mode 100644 provider/llamaswap/health_test.go diff --git a/README.md b/README.md index b267098..e0e08a9 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,9 @@ LLM_LS=llama-swaps://token@swap.example.com # https → TLS-fronted instance [llama-swap](https://github.com/mostlygeek/llama-swap) is a model-swapping proxy over llama.cpp. Its chat API is OpenAI-compatible (majordomo reuses the openai client), and the `*llamaswap.Provider` adds management methods -(`ListModels`/`Running`/`Unload`) plus image generation (see below). A cold +(`ListModels`/`Running`/`Unload`/`ListVoices`), a cheap liveness probe +(`Health`, GET /health — the proxy answers without touching a model), image +generation and editing, and speech synthesis/transcription (see below). A cold model swap can take many seconds — bound calls with a context deadline, not a client timeout. @@ -211,8 +213,9 @@ resp, err := m.Generate(ctx, majordomo.Request{ Text-to-image is a separate contract (`imagegen`) from chat, because it shares none of the message/tool/stream machinery. Generated images come back as `llm.ImagePart`, so they drop straight back into a chat turn. The first backend -is llama-swap (OpenAI `/v1/images/generations` → a stable-diffusion.cpp -upstream). +is llama-swap (the A1111-style `/sdapi/v1/txt2img` on a stable-diffusion.cpp +upstream — chosen over OpenAI `/v1/images/generations` because that route +ignores `seed` there). ```go ls := llamaswap.New(llamaswap.WithBaseURL("http://box.local:8080")) @@ -224,9 +227,45 @@ res, err := im.Generate(ctx, imagegen.Request{Prompt: "a red bicycle"}, // majordomo.UserParts(majordomo.Text("describe this"), res.Images[0]) ``` +Image-to-image editing is the optional `imagegen.Editor` interface (ADR-0018) +— llama-swap's image models implement it via `/sdapi/v1/img2img`: + +```go +ed := im.(imagegen.Editor) +res, err := ed.Edit(ctx, imagegen.EditRequest{ + Prompt: "make it night", + Init: res.Images[0], // any llm.ImagePart +}, imagegen.WithEditStrength(0.6)) // 0..1: how far to depart from Init +``` + `*llamaswap.Provider` also exposes management methods: `ListModels` (what llama-swap can serve), `Running` (what's loaded), and `Unload` (free a model). +## Speech: synthesis + transcription + +Text-to-speech and speech-to-text live in the `audio` package (ADR-0017), +mirroring imagegen: small `SpeechModel`/`TranscriptionModel` contracts, +zero values mean backend defaults, bytes in/out (never URLs). First backend: +llama-swap (OpenAI `/v1/audio/speech` + `/v1/audio/transcriptions` routed to +kokoro/whisper.cpp-style upstreams). + +```go +ls := llamaswap.New(llamaswap.WithBaseURL("http://box.local:8080")) + +sm, _ := ls.SpeechModel("kokoro") +speech, err := sm.Speak(ctx, audio.SpeechRequest{Input: "hello world"}, + audio.WithVoice("af_heart"), audio.WithFormat("mp3")) +// speech.Audio ([]byte) + speech.MIME ("audio/mpeg") + +tm, _ := ls.TranscriptionModel("whisper-large-v3-turbo") +tr, err := tm.Transcribe(ctx, audio.TranscriptionRequest{ + Audio: speech.Audio, MIME: speech.MIME, +}) +// tr.Text + +voices, err := ls.ListVoices(ctx, "kokoro") // []string of voice ids +``` + ## Tool calls ```go @@ -358,8 +397,9 @@ response as a single delta plus final event. ² llama-swap's chat is OpenAI-compatible and reuses the openai client, so these capabilities are present at the client level; whether a given call succeeds depends on the llama.cpp model llama-swap loads. llama-swap also provides -**image generation** (a separate `imagegen` axis, not shown above) and -management methods on `*llamaswap.Provider`. +**image generation + editing** (`imagegen`), **speech synthesis + +transcription** (`audio`) — separate axes, not shown above — plus a `Health` +probe and management methods on `*llamaswap.Provider`. Notes: Ollama has no native tool_choice — `"none"` drops the tools; `"required"`/named choices are best-effort ignored there. Ollama Cloud diff --git a/audio/audio.go b/audio/audio.go new file mode 100644 index 0000000..ec60db4 --- /dev/null +++ b/audio/audio.go @@ -0,0 +1,186 @@ +// Package audio is majordomo's canonical speech surface: text-to-speech +// (SpeechModel) and audio transcription (TranscriptionModel). Like imagegen, +// it is a deliberately separate contract from the llm package — synthesis and +// transcription share none of the chat message/tool/stream machinery, so they +// get their own small Provider/Model interfaces rather than overloading +// llm.Model (ADR-0017). +// +// Zero values mean "backend default" throughout, mirroring imagegen: an empty +// Voice uses the model's default voice, an empty Format the backend's default +// container, a zero Speed the natural rate. +// +// The first implementation is provider/llamaswap, which targets the OpenAI +// /v1/audio/speech and /v1/audio/transcriptions endpoints routed to +// kokoro/whisper.cpp-style upstreams. +package audio + +import "context" + +// SpeechRequest is a text-to-speech request. +type SpeechRequest struct { + // Input is the text to speak. + Input string + + // Voice selects the voice; "" = the model's default voice. + Voice string + + // Format is the audio container ("mp3", "wav", "opus", ...); + // "" = backend default. + Format string + + // Speed is the playback-rate multiplier; 0 = backend default (1.0). + Speed float64 +} + +// SpeechResult is the canonical synthesis result: raw audio bytes plus the +// MIME type reported (or implied) by the backend. +type SpeechResult struct { + // Audio is the encoded audio. + Audio []byte + + // MIME is the audio MIME type, e.g. "audio/mpeg". + MIME string + + // Raw is the provider-native response object, an escape hatch for + // provider-specific fields. May be nil; never required for normal use. + Raw any +} + +// SpeechOption mutates a SpeechRequest before it is sent. Options passed to +// Speak are applied to a copy, so a request value can be reused. +type SpeechOption func(*SpeechRequest) + +// WithVoice selects the voice. +func WithVoice(v string) SpeechOption { return func(r *SpeechRequest) { r.Voice = v } } + +// WithFormat sets the audio container format (e.g. "mp3", "wav"). +func WithFormat(f string) SpeechOption { return func(r *SpeechRequest) { r.Format = f } } + +// WithSpeed sets the playback-rate multiplier. +func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } } + +// Apply returns a copy of the request with all options applied. Providers +// call this once at the top of Speak. +func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// SpeechModel synthesizes speech from text. +type SpeechModel interface { + // Speak renders the request's input text as audio. + Speak(ctx context.Context, req SpeechRequest, opts ...SpeechOption) (*SpeechResult, error) +} + +// SpeechModelOption configures a SpeechModel at construction time. Reserved +// for future per-model settings; present so the interface is +// forward-compatible (mirrors imagegen.ModelOption). +type SpeechModelOption func(*SpeechModelConfig) + +// SpeechModelConfig carries per-model construction settings. +type SpeechModelConfig struct{} + +// ApplySpeechModelOptions folds options into a config. +func ApplySpeechModelOptions(opts []SpeechModelOption) SpeechModelConfig { + var cfg SpeechModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// SpeechProvider mints speech models bound to one backend. +type SpeechProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // SpeechModel returns a SpeechModel bound to the given id (passed through + // to the backend verbatim; no catalog validation). + SpeechModel(id string, opts ...SpeechModelOption) (SpeechModel, error) +} + +// TranscriptionRequest is a speech-to-text request. Audio is carried as bytes +// (never a URL), mirroring llm.ImagePart's bytes-only contract. +type TranscriptionRequest struct { + // Audio is the encoded audio to transcribe. + Audio []byte + + // MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend + // sniff it. + MIME string + + // Filename is the multipart filename hint some backends key their format + // detection on; "" derives one from MIME ("audio.mp3") or falls back to + // "audio". + Filename string + + // Language is a BCP-47/ISO-639 hint (e.g. "en"); "" = auto-detect. + Language string + + // Prompt is optional context or vocabulary to bias decoding; "" = none. + Prompt string +} + +// TranscriptionResult is the canonical transcription result. +type TranscriptionResult struct { + // Text is the transcript. + Text string + + // Raw is the provider-native response object. May be nil. + Raw any +} + +// TranscriptionOption mutates a TranscriptionRequest before it is sent. +type TranscriptionOption func(*TranscriptionRequest) + +// WithLanguage sets the language hint (e.g. "en"). +func WithLanguage(l string) TranscriptionOption { + return func(r *TranscriptionRequest) { r.Language = l } +} + +// WithPrompt sets the decoding context/vocabulary hint. +func WithPrompt(p string) TranscriptionOption { + return func(r *TranscriptionRequest) { r.Prompt = p } +} + +// Apply returns a copy of the request with all options applied. +func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// TranscriptionModel transcribes audio to text. +type TranscriptionModel interface { + // Transcribe converts the request's audio into text. + Transcribe(ctx context.Context, req TranscriptionRequest, opts ...TranscriptionOption) (*TranscriptionResult, error) +} + +// TranscriptionModelOption configures a TranscriptionModel at construction +// time. Reserved for future per-model settings. +type TranscriptionModelOption func(*TranscriptionModelConfig) + +// TranscriptionModelConfig carries per-model construction settings. +type TranscriptionModelConfig struct{} + +// ApplyTranscriptionModelOptions folds options into a config. +func ApplyTranscriptionModelOptions(opts []TranscriptionModelOption) TranscriptionModelConfig { + var cfg TranscriptionModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// TranscriptionProvider mints transcription models bound to one backend. +type TranscriptionProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // TranscriptionModel returns a TranscriptionModel bound to the given id + // (passed through to the backend verbatim; no catalog validation). + TranscriptionModel(id string, opts ...TranscriptionModelOption) (TranscriptionModel, error) +} diff --git a/audio/audio_test.go b/audio/audio_test.go new file mode 100644 index 0000000..2a96096 --- /dev/null +++ b/audio/audio_test.go @@ -0,0 +1,38 @@ +package audio + +import "testing" + +func TestSpeechRequestApply(t *testing.T) { + base := SpeechRequest{Input: "hello"} + got := base.Apply(WithVoice("af_heart"), WithFormat("wav"), WithSpeed(1.5)) + + if got.Input != "hello" { + t.Errorf("Input = %q, want %q", got.Input, "hello") + } + if got.Voice != "af_heart" || got.Format != "wav" || got.Speed != 1.5 { + t.Errorf("got = %+v", got) + } + + // Apply must not mutate the receiver (options apply to a copy). + if base.Voice != "" || base.Format != "" || base.Speed != 0 { + t.Errorf("base mutated: %+v", base) + } +} + +func TestTranscriptionRequestApply(t *testing.T) { + base := TranscriptionRequest{Audio: []byte{1, 2, 3}} + got := base.Apply(WithLanguage("en"), WithPrompt("names")) + + if got.Language != "en" || got.Prompt != "names" { + t.Errorf("got = %+v", got) + } + if base.Language != "" || base.Prompt != "" { + t.Errorf("base mutated: %+v", base) + } +} + +func TestApplyModelOptions(t *testing.T) { + // No options yet; just verify they return usable zero configs. + _ = ApplySpeechModelOptions(nil) + _ = ApplyTranscriptionModelOptions(nil) +} diff --git a/docs/adr/0017-audio-interfaces.md b/docs/adr/0017-audio-interfaces.md new file mode 100644 index 0000000..275608e --- /dev/null +++ b/docs/adr/0017-audio-interfaces.md @@ -0,0 +1,53 @@ +# ADR-0017: audio — canonical speech synthesis + transcription interfaces + +**Status:** Accepted — 2026-07-11 + +## Context + +mort is growing agent tools that speak (TTS) and transcribe audio through a +llama-swap host whose upstreams expose the OpenAI `/v1/audio/speech` and +`/v1/audio/transcriptions` endpoints (kokoro, chatterbox, whisper.cpp). Like +image generation before it (ADR-0016), speech shares none of the chat +contract's message/tool/stream machinery, and majordomo had no speech surface +— an earlier migration doc explicitly scoped transcription out of the llm +package. The same reasoning that produced `imagegen` applies. + +## Decision + +- One new canonical **leaf package `audio`** holding both directions — + synthesis and transcription — rather than two packages; they are one + modality and will share future types (voice metadata, audio formats). + Root re-exports mirror imagegen (`SpeechModel`, `SpeechProvider`, + `SpeechRequest`, `SpeechResult`, `TranscriptionModel`, ...). +- Minimal v1 surface, imagegen conventions throughout (zero value = backend + default, functional options + `Apply`, `Raw any` escape hatch): + - `SpeechRequest{ Input; Voice; Format; Speed }` → + `SpeechResult{ Audio []byte; MIME string; Raw }`; + `SpeechModel.Speak(ctx, req, ...opts)`; + `SpeechProvider.SpeechModel(id, ...)`. + - `TranscriptionRequest{ Audio []byte; MIME; Filename; Language; Prompt }` → + `TranscriptionResult{ Text string; Raw }`; + `TranscriptionModel.Transcribe(ctx, req, ...opts)`; + `TranscriptionProvider.TranscriptionModel(id, ...)`. +- **Bytes in/out, never URLs** — mirrors `llm.ImagePart`'s bytes-only + contract; fetching is the caller's concern. +- Providers are split (`SpeechProvider` vs `TranscriptionProvider`) so a + backend can implement either half; llamaswap implements both. +- First implementation: `provider/llamaswap` — `/v1/audio/speech` (JSON body, + raw audio response; MIME from Content-Type with a format-based fallback), + `/v1/audio/transcriptions` (multipart, `response_format=json`), plus + `ListVoices(ctx, model)` (GET `/v1/audio/voices?model=`, tolerant of the + string-list and object-list shapes upstreams use) as a llamaswap management + method, not part of the canonical contract. +- Out of scope for v1 (designed-for, deferred): streaming synthesis, + word-level timestamps/segments, translation, voice cloning inputs, and + registry-level DSN resolution for audio models. + +## Consequences + +- Speech is provider-agnostic from day one; an OpenAI or Google speech + backend implements the same interfaces. +- `SpeechResult.Audio` is a plain byte slice, so results flow into any file + store or attachment pipeline without a majordomo dependency. +- The `audio` package name is the modality, not the direction; if music/sfx + generation ever lands it has a home. diff --git a/docs/adr/0018-imagegen-editor.md b/docs/adr/0018-imagegen-editor.md new file mode 100644 index 0000000..f5c67e3 --- /dev/null +++ b/docs/adr/0018-imagegen-editor.md @@ -0,0 +1,36 @@ +# ADR-0018: imagegen.Editor — image-to-image as a separate optional interface + +**Status:** Accepted — 2026-07-11 + +## Context + +ADR-0016 shipped text-to-image and explicitly deferred img2img. mort's new +llama-swap media tools need "edit this image under this prompt" +(image-to-image with a denoising strength). Two shape questions: does Edit +belong on `imagegen.Model`, and which llama-swap endpoint carries it — +OpenAI-style `/v1/images/edits` (multipart) or A1111-style `/sdapi/v1/img2img` +(JSON)? + +## Decision + +- **`Editor` is a separate, optional interface** (`Edit(ctx, EditRequest, + ...EditOption) (*Result, error)`), not a new method on `Model`. Existing + `Model` implementations keep compiling; callers type-assert + (`m.(imagegen.Editor)`) or require the capability explicitly. +- `EditRequest` = the generation knobs (prompt, N, size, steps, cfg, negative + prompt, sampler, seed) plus `Init Image` (required) and `Strength *float64` + (denoising strength in [0,1]; nil = backend default). Same option/Apply + conventions; result type is the shared `Result`. +- llama-swap implements it via **`/sdapi/v1/img2img`**, not + `/v1/images/edits`: the same sd-server build that ignores `seed` on the + OpenAI images route (the reason txt2img went SDAPI in ADR-0016's + implementation) applies; the JSON shape is txt2img's plus + `init_images: [""]` + `denoising_strength`, so it reuses `doJSON` + verbatim, where the OpenAI route is multipart. + +## Consequences + +- Backends that can't edit simply don't implement `Editor`; no stub methods. +- The init image travels base64-inline in JSON (~33% overhead) — acceptable at + chat-image sizes; a future backend needing multipart can still satisfy the + same interface. diff --git a/docs/adr/README.md b/docs/adr/README.md index 9261b1d..6818feb 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -20,3 +20,5 @@ One decision per file, append-only; supersede rather than rewrite. | [0014](0014-conversion-driven-extensions.md) | Conversion-driven extensions (resolvers, typed tools, hooks, ops controls) | Accepted | | [0015](0015-llama-swap-provider.md) | llama-swap provider — reuse openai for chat, tailored management + image | Accepted | | [0016](0016-imagegen-interface.md) | imagegen — a canonical text-to-image interface | Accepted | +| [0017](0017-audio-interfaces.md) | audio — canonical speech synthesis + transcription interfaces | Accepted | +| [0018](0018-imagegen-editor.md) | imagegen.Editor — image-to-image as a separate optional interface | Accepted | diff --git a/imagegen/edit.go b/imagegen/edit.go new file mode 100644 index 0000000..cecdbaa --- /dev/null +++ b/imagegen/edit.go @@ -0,0 +1,91 @@ +package imagegen + +import "context" + +// EditRequest is an image-to-image (edit) request: a prompt applied to an +// initial image. As with Request, zero values mean "backend default" +// (ADR-0018). +type EditRequest struct { + // Prompt is the text description of the desired edit. + Prompt string + + // Init is the initial image the edit starts from. Required. + Init Image + + // Strength is the denoising strength in [0,1] — how far the result may + // depart from Init (0 = return the input, 1 = ignore it); nil = backend + // default. + Strength *float64 + + // N is the number of images to generate; 0 = provider default. + N int + + // Size is the requested resolution, e.g. "1024x1024"; "" = provider + // default (usually the init image's own resolution). + Size string + + // Steps is the number of diffusion steps; nil = backend default. + Steps *int + + // CFGScale is the classifier-free-guidance scale; nil = backend default. + CFGScale *float64 + + // NegativePrompt steers generation away from concepts; "" = none. + NegativePrompt string + + // Sampler selects the sampling method (e.g. "euler", "euler_a"); + // "" = backend default. + Sampler string + + // Seed fixes the RNG seed for reproducible output; nil = random. + Seed *int64 +} + +// EditOption mutates an EditRequest before it is sent. Options passed to Edit +// are applied to a copy of the request, so an EditRequest value can be reused. +type EditOption func(*EditRequest) + +// WithEditStrength sets the denoising strength in [0,1]. +func WithEditStrength(s float64) EditOption { return func(r *EditRequest) { r.Strength = &s } } + +// WithEditN sets the number of images to generate. +func WithEditN(n int) EditOption { return func(r *EditRequest) { r.N = n } } + +// WithEditSize sets the requested resolution (e.g. "1024x1024"). +func WithEditSize(size string) EditOption { return func(r *EditRequest) { r.Size = size } } + +// WithEditSteps overrides the number of diffusion steps. +func WithEditSteps(n int) EditOption { return func(r *EditRequest) { r.Steps = &n } } + +// WithEditCFGScale overrides the classifier-free-guidance scale. +func WithEditCFGScale(s float64) EditOption { return func(r *EditRequest) { r.CFGScale = &s } } + +// WithEditNegativePrompt sets a negative prompt. +func WithEditNegativePrompt(s string) EditOption { + return func(r *EditRequest) { r.NegativePrompt = s } +} + +// WithEditSampler overrides the sampling method. +func WithEditSampler(s string) EditOption { return func(r *EditRequest) { r.Sampler = s } } + +// WithEditSeed fixes the RNG seed for reproducible output. +func WithEditSeed(seed int64) EditOption { return func(r *EditRequest) { r.Seed = &seed } } + +// Apply returns a copy of the request with all options applied. Providers +// call this once at the top of Edit. +func (r EditRequest) Apply(opts ...EditOption) EditRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// Editor is the image-to-image surface. It is a separate, optional interface +// rather than a method on Model so existing Model implementations keep +// compiling; callers type-assert (`m.(imagegen.Editor)`) or require it +// explicitly. +type Editor interface { + // Edit produces one or more images derived from the request's init image + // under the request's prompt. + Edit(ctx context.Context, req EditRequest, opts ...EditOption) (*Result, error) +} diff --git a/imagegen/edit_test.go b/imagegen/edit_test.go new file mode 100644 index 0000000..164e700 --- /dev/null +++ b/imagegen/edit_test.go @@ -0,0 +1,26 @@ +package imagegen + +import "testing" + +func TestEditRequestApply(t *testing.T) { + base := EditRequest{Prompt: "make it night", Init: Image{MIME: "image/png", Data: []byte{1}}} + got := base.Apply(WithEditStrength(0.7), WithEditN(2), WithEditSeed(42)) + + if got.Prompt != "make it night" || len(got.Init.Data) != 1 { + t.Errorf("got = %+v", got) + } + if got.Strength == nil || *got.Strength != 0.7 { + t.Errorf("Strength = %v, want 0.7", got.Strength) + } + if got.N != 2 { + t.Errorf("N = %d, want 2", got.N) + } + if got.Seed == nil || *got.Seed != 42 { + t.Errorf("Seed = %v, want 42", got.Seed) + } + + // Apply must not mutate the receiver (options apply to a copy). + if base.Strength != nil || base.N != 0 || base.Seed != nil { + t.Errorf("base mutated: %+v", base) + } +} diff --git a/majordomo.go b/majordomo.go index b21226a..54b4e84 100644 --- a/majordomo.go +++ b/majordomo.go @@ -26,6 +26,7 @@ import ( "encoding/json" "sync" + "gitea.stevedudenhoeffer.com/steve/majordomo/audio" "gitea.stevedudenhoeffer.com/steve/majordomo/imagegen" "gitea.stevedudenhoeffer.com/steve/majordomo/llm" ) @@ -67,6 +68,26 @@ type ( ImageResult = imagegen.Result ImageOption = imagegen.Option ImageModelOption = imagegen.ModelOption + ImageEditor = imagegen.Editor + ImageEditRequest = imagegen.EditRequest + ImageEditOption = imagegen.EditOption +) + +// Re-exported canonical speech types. See the audio package for +// documentation. Speech synthesis and transcription are separate contracts +// from llm, mirroring imagegen (ADR-0017); the first backend is +// provider/llamaswap. +type ( + SpeechModel = audio.SpeechModel + SpeechProvider = audio.SpeechProvider + SpeechRequest = audio.SpeechRequest + SpeechResult = audio.SpeechResult + SpeechOption = audio.SpeechOption + TranscriptionModel = audio.TranscriptionModel + TranscriptionProvider = audio.TranscriptionProvider + TranscriptionRequest = audio.TranscriptionRequest + TranscriptionResult = audio.TranscriptionResult + TranscriptionOption = audio.TranscriptionOption ) // Re-exported role and finish-reason constants. diff --git a/progress.md b/progress.md index d441216..3c718b7 100644 --- a/progress.md +++ b/progress.md @@ -248,3 +248,19 @@ alias-in-chain failover, permanent-policy override) and wires anything the tests flush out. **Next:** Phase 2 — exhaustive health/chain test matrix. + +## 2026-07-11 — audio surfaces + image editing (ADR-0017, ADR-0018) + +- New leaf package `audio`: `SpeechModel`/`SpeechProvider` (TTS) and + `TranscriptionModel`/`TranscriptionProvider` (STT), imagegen conventions + (zero value = backend default, options + Apply, bytes in/out). Root + re-exports added. +- `imagegen.Editor` — optional img2img interface (`EditRequest` with + `Init Image` + `Strength`); shared `Result`. +- provider/llamaswap implements all of it: `/v1/audio/speech`, + `/v1/audio/transcriptions` (multipart), `ListVoices` + (`/v1/audio/voices?model=`, shape-tolerant), `/sdapi/v1/img2img` + (txt2img wire + init_images/denoising_strength, shared decode), and a + cheap `Health(ctx)` probe (GET /health) for often-offline hosts. +- Hermetic httptest coverage for every new wire shape + validation errors. +- Consumer: mort's llamaswap media tool cluster (status/image/TTS/STT tools). diff --git a/provider/llamaswap/audio.go b/provider/llamaswap/audio.go new file mode 100644 index 0000000..2308680 --- /dev/null +++ b/provider/llamaswap/audio.go @@ -0,0 +1,278 @@ +package llamaswap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "net/url" + "strings" + + majaudio "gitea.stevedudenhoeffer.com/steve/majordomo/audio" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" +) + +// SpeechModel implements audio.SpeechProvider, binding a text-to-speech model +// served by llama-swap (routed to a kokoro/chatterbox-style OpenAI-compatible +// upstream). The id is passed through verbatim and selects which upstream +// llama-swap loads. +func (p *Provider) SpeechModel(id string, opts ...majaudio.SpeechModelOption) (majaudio.SpeechModel, error) { + if p.baseURL == "" { + return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + } + _ = majaudio.ApplySpeechModelOptions(opts) + return &speechModel{p: p, id: id}, nil +} + +type speechModel struct { + p *Provider + id string +} + +// speechRequest is the OpenAI /v1/audio/speech request shape. llama-swap +// routes by the `model` field in the body. +type speechRequest struct { + Model string `json:"model"` + Input string `json:"input"` + Voice string `json:"voice,omitempty"` + ResponseFormat string `json:"response_format,omitempty"` + Speed float64 `json:"speed,omitempty"` +} + +// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech. +func (m *speechModel) Speak(ctx context.Context, req majaudio.SpeechRequest, opts ...majaudio.SpeechOption) (*majaudio.SpeechResult, error) { + req = req.Apply(opts...) + if strings.TrimSpace(req.Input) == "" { + return nil, fmt.Errorf("%w: speech synthesis requires input text", llm.ErrUnsupported) + } + wire := speechRequest{ + Model: m.id, + Input: req.Input, + Voice: req.Voice, + ResponseFormat: req.Format, + Speed: req.Speed, + } + body, err := json.Marshal(wire) + if err != nil { + return nil, fmt.Errorf("llama-swap: encode speech request: %w", err) + } + audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, "/v1/audio/speech", m.id, "application/json", bytes.NewReader(body)) + if err != nil { + return nil, err + } + if len(audioBytes) == 0 { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech response contained no audio"} + } + return &majaudio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil +} + +// speechMIME resolves the result MIME type: the response Content-Type when it +// is a concrete audio type, else a mapping from the requested format, else +// audio/mpeg (the OpenAI endpoint's default container is mp3). +func speechMIME(contentType, format string) string { + if mt, _, err := mime.ParseMediaType(contentType); err == nil { + if strings.HasPrefix(mt, "audio/") || strings.HasPrefix(mt, "video/") { + return mt + } + } + switch strings.ToLower(strings.TrimSpace(format)) { + case "", "mp3": + return "audio/mpeg" + case "wav": + return "audio/wav" + case "opus": + return "audio/ogg" + case "aac": + return "audio/aac" + case "flac": + return "audio/flac" + case "pcm": + return "audio/pcm" + default: + return "audio/mpeg" + } +} + +// TranscriptionModel implements audio.TranscriptionProvider, binding a +// speech-to-text model served by llama-swap (routed to a whisper.cpp-style +// upstream). +func (p *Provider) TranscriptionModel(id string, opts ...majaudio.TranscriptionModelOption) (majaudio.TranscriptionModel, error) { + if p.baseURL == "" { + return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + } + _ = majaudio.ApplyTranscriptionModelOptions(opts) + return &transcriptionModel{p: p, id: id}, nil +} + +type transcriptionModel struct { + p *Provider + id string +} + +// Transcribe implements audio.TranscriptionModel via POST +// {base}/v1/audio/transcriptions (multipart/form-data — llama-swap routes by +// the `model` form field). +func (m *transcriptionModel) Transcribe(ctx context.Context, req majaudio.TranscriptionRequest, opts ...majaudio.TranscriptionOption) (*majaudio.TranscriptionResult, error) { + req = req.Apply(opts...) + if len(req.Audio) == 0 { + return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported) + } + + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + fw, err := w.CreateFormFile("file", transcriptionFilename(req)) + if err != nil { + return nil, fmt.Errorf("llama-swap: build transcription form: %w", err) + } + if _, err := fw.Write(req.Audio); err != nil { + return nil, fmt.Errorf("llama-swap: build transcription form: %w", err) + } + fields := map[string]string{ + "model": m.id, + "language": req.Language, + "prompt": req.Prompt, + "response_format": "json", + } + for k, v := range fields { + if v == "" { + continue + } + if err := w.WriteField(k, v); err != nil { + return nil, fmt.Errorf("llama-swap: build transcription form: %w", err) + } + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("llama-swap: build transcription form: %w", err) + } + + raw, _, err := m.p.doRaw(ctx, http.MethodPost, "/v1/audio/transcriptions", m.id, w.FormDataContentType(), &buf) + if err != nil { + return nil, err + } + var out struct { + Text string `json:"text"` + } + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("llama-swap: decode transcription response: %w", err) + } + return &majaudio.TranscriptionResult{Text: out.Text, Raw: json.RawMessage(raw)}, nil +} + +// transcriptionFilename picks the multipart filename hint: the caller's, else +// one derived from the MIME subtype ("audio.mp3"), else "audio". +func transcriptionFilename(req majaudio.TranscriptionRequest) string { + if req.Filename != "" { + return req.Filename + } + switch strings.ToLower(req.MIME) { + case "audio/mpeg", "audio/mp3": + return "audio.mp3" + case "audio/wav", "audio/x-wav", "audio/wave": + return "audio.wav" + case "audio/ogg", "audio/opus": + return "audio.ogg" + case "audio/flac", "audio/x-flac": + return "audio.flac" + case "audio/mp4", "audio/m4a", "audio/x-m4a": + return "audio.m4a" + case "audio/webm", "video/webm": + return "audio.webm" + default: + return "audio" + } +} + +// ListVoices returns the voices a TTS model offers (GET +// /v1/audio/voices?model=...). The decode is tolerant: upstreams answer with +// either a bare string list or a list of {id|name} objects. +func (p *Provider) ListVoices(ctx context.Context, model string) ([]string, error) { + if model == "" { + return nil, fmt.Errorf("llama-swap: ListVoices requires a model id") + } + raw, _, err := p.doRaw(ctx, http.MethodGet, "/v1/audio/voices?model="+url.QueryEscape(model), model, "", nil) + if err != nil { + return nil, err + } + return parseVoices(raw) +} + +// parseVoices extracts voice names from the various shapes upstreams use: +// {"voices":[...]} or {"data":[...]} envelopes (or a bare array), holding +// either strings or objects keyed by id/name/voice_id. +func parseVoices(raw []byte) ([]string, error) { + var env struct { + Voices json.RawMessage `json:"voices"` + Data json.RawMessage `json:"data"` + } + list := json.RawMessage(raw) + if err := json.Unmarshal(raw, &env); err == nil { + if len(env.Voices) > 0 { + list = env.Voices + } else if len(env.Data) > 0 { + list = env.Data + } + } + + var names []string + if err := json.Unmarshal(list, &names); err == nil { + return names, nil + } + // A failed decode above may have partially populated names — start fresh. + names = nil + var objs []struct { + ID string `json:"id"` + Name string `json:"name"` + VoiceID string `json:"voice_id"` + } + if err := json.Unmarshal(list, &objs); err == nil { + for _, o := range objs { + switch { + case o.ID != "": + names = append(names, o.ID) + case o.Name != "": + names = append(names, o.Name) + case o.VoiceID != "": + names = append(names, o.VoiceID) + } + } + return names, nil + } + return nil, fmt.Errorf("llama-swap: unrecognized voices payload shape") +} + +// doRaw performs a request to a llama-swap endpoint and returns the raw +// response body and its Content-Type — the sibling of doJSON for endpoints +// whose success payload is not JSON (audio bytes) or whose shape varies. +// contentType sets the request Content-Type when body is non-nil. +func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType string, body io.Reader) ([]byte, string, error) { + if p.baseURL == "" { + return nil, "", fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + } + req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, body) + if err != nil { + return nil, "", fmt.Errorf("llama-swap: build request: %w", err) + } + if body != nil && contentType != "" { + req.Header.Set("Content-Type", contentType) + } + if p.token != "" { + req.Header.Set("Authorization", "Bearer "+p.token) + } + resp, err := p.client.Do(req) + if err != nil { + return nil, "", fmt.Errorf("llama-swap: do request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return nil, "", p.apiError(resp, model) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes)) + if err != nil { + return nil, "", fmt.Errorf("llama-swap: read response: %w", err) + } + return data, resp.Header.Get("Content-Type"), nil +} diff --git a/provider/llamaswap/audio_test.go b/provider/llamaswap/audio_test.go new file mode 100644 index 0000000..ac8bec9 --- /dev/null +++ b/provider/llamaswap/audio_test.go @@ -0,0 +1,247 @@ +package llamaswap + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "gitea.stevedudenhoeffer.com/steve/majordomo/audio" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" +) + +func TestSpeak(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/audio/speech" { + t.Errorf("path = %q", r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer tok" { + t.Errorf("auth = %q", r.Header.Get("Authorization")) + } + _ = json.NewDecoder(r.Body).Decode(&gotBody) + w.Header().Set("Content-Type", "audio/mpeg") + _, _ = w.Write([]byte("MP3BYTES")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithToken("tok"), WithHTTPClient(srv.Client())) + sm, err := p.SpeechModel("kokoro") + if err != nil { + t.Fatalf("SpeechModel: %v", err) + } + res, err := sm.Speak(context.Background(), + audio.SpeechRequest{Input: "hello world"}, + audio.WithVoice("af_heart"), audio.WithFormat("mp3"), audio.WithSpeed(1.2), + ) + if err != nil { + t.Fatalf("Speak: %v", err) + } + if string(res.Audio) != "MP3BYTES" || res.MIME != "audio/mpeg" { + t.Errorf("result = %q %q", res.Audio, res.MIME) + } + want := map[string]any{"model": "kokoro", "input": "hello world", "voice": "af_heart", "response_format": "mp3", "speed": 1.2} + for k, w := range want { + if gotBody[k] != w { + t.Errorf("%s = %v, want %v", k, gotBody[k], w) + } + } +} + +func TestSpeakDefaultsOmitted(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte("x")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + sm, _ := p.SpeechModel("kokoro") + res, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: "hi"}) + if err != nil { + t.Fatalf("Speak: %v", err) + } + for _, k := range []string{"voice", "response_format", "speed"} { + if v, ok := gotBody[k]; ok { + t.Errorf("unset request sent %q = %v, want omitted", k, v) + } + } + // No usable Content-Type and no requested format → mp3 default. + if res.MIME != "audio/mpeg" { + t.Errorf("MIME = %q, want audio/mpeg fallback", res.MIME) + } +} + +func TestSpeakEmptyInput(t *testing.T) { + p := New(WithBaseURL("http://example.invalid")) + sm, _ := p.SpeechModel("kokoro") + if _, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: " "}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("err = %v, want ErrUnsupported", err) + } +} + +func TestSpeechMIME(t *testing.T) { + cases := []struct { + contentType, format, want string + }{ + {"audio/ogg; codecs=opus", "mp3", "audio/ogg"}, // concrete header wins + {"application/octet-stream", "wav", "audio/wav"}, + {"", "", "audio/mpeg"}, + {"", "opus", "audio/ogg"}, + {"text/plain", "flac", "audio/flac"}, + } + for _, tc := range cases { + if got := speechMIME(tc.contentType, tc.format); got != tc.want { + t.Errorf("speechMIME(%q, %q) = %q, want %q", tc.contentType, tc.format, got, tc.want) + } + } +} + +func TestTranscribe(t *testing.T) { + var gotFields map[string]string + var gotFile []byte + var gotFilename string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/audio/transcriptions" { + t.Errorf("path = %q", r.URL.Path) + } + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + gotFields = map[string]string{} + for k, v := range r.MultipartForm.Value { + gotFields[k] = v[0] + } + f, hdr, err := r.FormFile("file") + if err != nil { + t.Fatalf("form file: %v", err) + } + defer f.Close() + gotFile, _ = io.ReadAll(f) + gotFilename = hdr.Filename + _, _ = w.Write([]byte(`{"text":"hello there"}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + tm, err := p.TranscriptionModel("whisper") + if err != nil { + t.Fatalf("TranscriptionModel: %v", err) + } + res, err := tm.Transcribe(context.Background(), + audio.TranscriptionRequest{Audio: []byte("AUDIO"), MIME: "audio/mpeg"}, + audio.WithLanguage("en"), audio.WithPrompt("robot names"), + ) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if res.Text != "hello there" { + t.Errorf("text = %q", res.Text) + } + if string(gotFile) != "AUDIO" || gotFilename != "audio.mp3" { + t.Errorf("file = %q name = %q", gotFile, gotFilename) + } + want := map[string]string{"model": "whisper", "language": "en", "prompt": "robot names", "response_format": "json"} + for k, w := range want { + if gotFields[k] != w { + t.Errorf("%s = %q, want %q", k, gotFields[k], w) + } + } +} + +func TestTranscribeEmptyAudio(t *testing.T) { + p := New(WithBaseURL("http://example.invalid")) + tm, _ := p.TranscriptionModel("whisper") + if _, err := tm.Transcribe(context.Background(), audio.TranscriptionRequest{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("err = %v, want ErrUnsupported", err) + } +} + +func TestListVoices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/audio/voices" { + t.Errorf("path = %q", r.URL.Path) + } + if got := r.URL.Query().Get("model"); got != "kokoro" { + t.Errorf("model = %q", got) + } + _, _ = w.Write([]byte(`{"voices":["af_heart","af_bella"]}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + voices, err := p.ListVoices(context.Background(), "kokoro") + if err != nil { + t.Fatalf("ListVoices: %v", err) + } + if !reflect.DeepEqual(voices, []string{"af_heart", "af_bella"}) { + t.Errorf("voices = %v", voices) + } + + if _, err := p.ListVoices(context.Background(), ""); err == nil { + t.Error("empty model: want error") + } +} + +func TestParseVoicesShapes(t *testing.T) { + cases := []struct { + name string + raw string + want []string + }{ + {"envelope strings", `{"voices":["a","b"]}`, []string{"a", "b"}}, + {"bare array", `["a","b"]`, []string{"a", "b"}}, + {"objects by id", `{"voices":[{"id":"a"},{"id":"b"}]}`, []string{"a", "b"}}, + {"objects by name", `{"data":[{"name":"a"},{"voice_id":"b"}]}`, []string{"a", "b"}}, + } + for _, tc := range cases { + got, err := parseVoices([]byte(tc.raw)) + if err != nil { + t.Errorf("%s: %v", tc.name, err) + continue + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("%s: got %v, want %v", tc.name, got, tc.want) + } + } + if _, err := parseVoices([]byte(`"just a string"`)); err == nil { + t.Error("unparseable shape: want error") + } +} + +func TestAudioAPIError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":{"message":"model is loading"}}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + sm, _ := p.SpeechModel("kokoro") + _, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: "x"}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %T %v, want *llm.APIError", err, err) + } + if apiErr.Status != http.StatusServiceUnavailable || apiErr.Message != "model is loading" || apiErr.Model != "kokoro" { + t.Errorf("apiErr = %+v", apiErr) + } +} + +func TestAudioNoBaseURL(t *testing.T) { + p := New() + if _, err := p.SpeechModel("kokoro"); err == nil { + t.Error("SpeechModel: want error without base URL") + } + if _, err := p.TranscriptionModel("whisper"); err == nil { + t.Error("TranscriptionModel: want error without base URL") + } + if _, err := p.ListVoices(context.Background(), "kokoro"); err == nil { + t.Error("ListVoices: want error without base URL") + } +} diff --git a/provider/llamaswap/edit_test.go b/provider/llamaswap/edit_test.go new file mode 100644 index 0000000..a8b765d --- /dev/null +++ b/provider/llamaswap/edit_test.go @@ -0,0 +1,110 @@ +package llamaswap + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "gitea.stevedudenhoeffer.com/steve/majordomo/imagegen" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" +) + +func editInit(t *testing.T) imagegen.Image { + t.Helper() + raw, err := base64.StdEncoding.DecodeString(onePixelPNG) + if err != nil { + t.Fatalf("decode fixture: %v", err) + } + return imagegen.Image{MIME: "image/png", Data: raw} +} + +func TestImageEdit(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/sdapi/v1/img2img" { + t.Errorf("path = %q", r.URL.Path) + } + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + im, _ := p.ImageModel("sd") + ed, ok := im.(imagegen.Editor) + if !ok { + t.Fatal("imageModel does not implement imagegen.Editor") + } + + res, err := ed.Edit(context.Background(), + imagegen.EditRequest{Prompt: "make it night", Init: editInit(t)}, + imagegen.WithEditStrength(0.6), + ) + if err != nil { + t.Fatalf("Edit: %v", err) + } + if len(res.Images) != 1 || res.Images[0].MIME != "image/png" { + t.Fatalf("images = %+v", res.Images) + } + + if gotBody["model"] != "sd" || gotBody["prompt"] != "make it night" { + t.Errorf("model/prompt = %v/%v", gotBody["model"], gotBody["prompt"]) + } + inits, ok := gotBody["init_images"].([]any) + if !ok || len(inits) != 1 || inits[0] != onePixelPNG { + t.Errorf("init_images = %v, want the b64 fixture", gotBody["init_images"]) + } + if gotBody["denoising_strength"] != 0.6 { + t.Errorf("denoising_strength = %v, want 0.6", gotBody["denoising_strength"]) + } +} + +func TestImageEditOmitsUnsetOverrides(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&gotBody) + _, _ = w.Write([]byte(`{"images":["` + onePixelPNG + `"]}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + im, _ := p.ImageModel("sd") + ed := im.(imagegen.Editor) + if _, err := ed.Edit(context.Background(), imagegen.EditRequest{Prompt: "x", Init: editInit(t)}); err != nil { + t.Fatalf("Edit: %v", err) + } + for _, k := range []string{"denoising_strength", "steps", "cfg_scale", "negative_prompt", "sample_method", "seed", "width", "height"} { + if v, ok := gotBody[k]; ok { + t.Errorf("unset request sent %q = %v, want omitted", k, v) + } + } +} + +func TestImageEditValidation(t *testing.T) { + p := New(WithBaseURL("http://example.invalid")) + im, _ := p.ImageModel("sd") + ed := im.(imagegen.Editor) + + cases := []struct { + name string + req imagegen.EditRequest + }{ + {"empty prompt", imagegen.EditRequest{Prompt: " ", Init: imagegen.Image{Data: []byte{1}}}}, + {"missing init", imagegen.EditRequest{Prompt: "x"}}, + {"negative N", imagegen.EditRequest{Prompt: "x", Init: imagegen.Image{Data: []byte{1}}, N: -1}}, + } + for _, tc := range cases { + if _, err := ed.Edit(context.Background(), tc.req); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("%s: err = %v, want ErrUnsupported", tc.name, err) + } + } + + bad := 1.5 + if _, err := ed.Edit(context.Background(), imagegen.EditRequest{Prompt: "x", Init: imagegen.Image{Data: []byte{1}}, Strength: &bad}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("out-of-range strength: err = %v, want ErrUnsupported", err) + } +} diff --git a/provider/llamaswap/health.go b/provider/llamaswap/health.go new file mode 100644 index 0000000..50a679a --- /dev/null +++ b/provider/llamaswap/health.go @@ -0,0 +1,45 @@ +package llamaswap + +import ( + "context" + "fmt" + "io" + "net/http" +) + +// Health reports whether the llama-swap instance is reachable (GET +// {base}/health, which answers "OK" without touching any model). A nil error +// means the proxy itself is up — it says nothing about how long a subsequent +// request will take (a cold model swap can still block for minutes). Bound +// the probe with a short context deadline; the client has no timeout by +// design. +func (p *Provider) Health(ctx context.Context) error { + if p.baseURL == "" { + return fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.baseURL+"/health", nil) + if err != nil { + return fmt.Errorf("llama-swap: build request: %w", err) + } + if p.token != "" { + req.Header.Set("Authorization", "Bearer "+p.token) + } + resp, err := p.client.Do(req) + if err != nil { + return fmt.Errorf("llama-swap: health probe: %w", err) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<10)) + if resp.StatusCode/100 != 2 { + return &apiHealthError{status: resp.StatusCode} + } + return nil +} + +// apiHealthError distinguishes "reachable but unhealthy" from transport +// failure without pulling in the llm error taxonomy for a probe. +type apiHealthError struct{ status int } + +func (e *apiHealthError) Error() string { + return fmt.Sprintf("llama-swap: health endpoint returned status %d", e.status) +} diff --git a/provider/llamaswap/health_test.go b/provider/llamaswap/health_test.go new file mode 100644 index 0000000..d5425ee --- /dev/null +++ b/provider/llamaswap/health_test.go @@ -0,0 +1,59 @@ +package llamaswap + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestHealth(t *testing.T) { + var gotPath, gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + _, _ = w.Write([]byte("OK")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithToken("tok"), WithHTTPClient(srv.Client())) + if err := p.Health(context.Background()); err != nil { + t.Fatalf("Health: %v", err) + } + if gotPath != "/health" || gotAuth != "Bearer tok" { + t.Errorf("path/auth = %q/%q", gotPath, gotAuth) + } +} + +func TestHealthUnhealthyStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + err := p.Health(context.Background()) + if err == nil || !strings.Contains(err.Error(), "502") { + t.Errorf("err = %v, want status-502 error", err) + } +} + +func TestHealthUnreachable(t *testing.T) { + // A closed server: transport error, not an HTTP status. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + url := srv.URL + srv.Close() + + p := New(WithBaseURL(url)) + if err := p.Health(context.Background()); err == nil { + t.Error("want error for unreachable host") + } +} + +func TestHealthNoBaseURL(t *testing.T) { + p := New() + if err := p.Health(context.Background()); err == nil { + t.Error("want error without base URL") + } +} diff --git a/provider/llamaswap/image.go b/provider/llamaswap/image.go index 2fe8914..c5edd19 100644 --- a/provider/llamaswap/image.go +++ b/provider/llamaswap/image.go @@ -85,8 +85,13 @@ func (m *imageModel) Generate(ctx context.Context, req imagegen.Request, opts .. if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/txt2img", m.id, &wire, &resp); err != nil { return nil, err } + return decodeImages(m.p.name, m.id, &resp) +} - out := &imagegen.Result{Raw: &resp} +// decodeImages converts an SDAPI response's base64 images into an +// imagegen.Result, erroring when nothing decodable came back. +func decodeImages(provider, model string, resp *txt2imgResponse) (*imagegen.Result, error) { + out := &imagegen.Result{Raw: resp} for i, b64 := range resp.Images { if b64 == "" { continue @@ -99,14 +104,70 @@ func (m *imageModel) Generate(ctx context.Context, req imagegen.Request, opts .. } if len(out.Images) == 0 { return nil, &llm.APIError{ - Provider: m.p.name, - Model: m.id, + Provider: provider, + Model: model, Message: "image response contained no images", } } return out, nil } +// img2imgRequest is the stable-diffusion.cpp sd-server A1111 request shape +// (POST /sdapi/v1/img2img): txt2img's fields plus the init image(s) and +// denoising strength. Same endpoint-family choice as txt2img — the OpenAI +// /v1/images/edits route is multipart and drops `seed` on this sd-server +// build, while the SDAPI shape reuses doJSON and keeps seed parity. +type img2imgRequest struct { + txt2imgRequest + InitImages []string `json:"init_images"` + DenoisingStrength *float64 `json:"denoising_strength,omitempty"` +} + +// Edit implements imagegen.Editor via POST {base}/sdapi/v1/img2img. +func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ...imagegen.EditOption) (*imagegen.Result, error) { + req = req.Apply(opts...) + if strings.TrimSpace(req.Prompt) == "" { + return nil, fmt.Errorf("%w: image edit requires a prompt", llm.ErrUnsupported) + } + if len(req.Init.Data) == 0 { + return nil, fmt.Errorf("%w: image edit requires an init image", llm.ErrUnsupported) + } + if req.N < 0 { + return nil, fmt.Errorf("%w: image count N must be >= 0, got %d", llm.ErrUnsupported, req.N) + } + if req.Strength != nil && (*req.Strength < 0 || *req.Strength > 1) { + return nil, fmt.Errorf("%w: edit strength must be in [0,1], got %g", llm.ErrUnsupported, *req.Strength) + } + + width, height, err := parseSize(req.Size) + if err != nil { + return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err) + } + + wire := img2imgRequest{ + txt2imgRequest: txt2imgRequest{ + Model: m.id, + Prompt: req.Prompt, + NegativePrompt: req.NegativePrompt, + Seed: req.Seed, + Steps: req.Steps, + CFGScale: req.CFGScale, + Width: width, + Height: height, + SampleMethod: req.Sampler, + BatchCount: req.N, + }, + InitImages: []string{base64.StdEncoding.EncodeToString(req.Init.Data)}, + DenoisingStrength: req.Strength, + } + + var resp txt2imgResponse + if err := m.p.doJSON(ctx, http.MethodPost, "/sdapi/v1/img2img", m.id, &wire, &resp); err != nil { + return nil, err + } + return decodeImages(m.p.name, m.id, &resp) +} + // parseSize splits a "WxH" string into width/height pointers. "" yields // (nil, nil) so the model's own default resolution applies. func parseSize(size string) (*int, *int, error) { From 9c0ac1d60bc0c3cf7d51c7d058185112a1a9bb61 Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Sat, 11 Jul 2026 23:46:23 -0400 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20address=20gadfly=20review=20?= =?UTF-8?q?=E2=80=94=20filename=20sanitization,=20truncation=20guard,=20sh?= =?UTF-8?q?ared=20plumbing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Transcribe: sanitize the caller-supplied multipart filename (CR/LF would inject Content-Disposition headers; upload metadata is untrusted), always send the required model/response_format fields, parse MIME parameters before extension matching, and give audio/opus its own .opus extension. - doRaw: a response larger than maxResponseBytes is now an error, not a silent truncation. - Shared plumbing: requireBaseURL() + newRequest() helpers replace the 7x-duplicated guard/error string and the triplicated request building across doJSON/doRaw/Health. - Health: non-2xx now returns *llm.APIError (package convention, programmatically distinguishable from transport failure) instead of a one-off unexported error type. - Speak: reject negative Speed; speechMIME no longer accepts video/* Content-Types. - image.go: Generate/Edit share one sdWire validate+map helper. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AXQxVhXBw8PwFAtsVrXSmj --- provider/llamaswap/audio.go | 119 ++++++++++++++++++-------------- provider/llamaswap/health.go | 31 ++++----- provider/llamaswap/image.go | 80 +++++++++------------ provider/llamaswap/llamaswap.go | 53 +++++++++----- 4 files changed, 155 insertions(+), 128 deletions(-) diff --git a/provider/llamaswap/audio.go b/provider/llamaswap/audio.go index 2308680..77ffc6a 100644 --- a/provider/llamaswap/audio.go +++ b/provider/llamaswap/audio.go @@ -12,7 +12,7 @@ import ( "net/url" "strings" - majaudio "gitea.stevedudenhoeffer.com/steve/majordomo/audio" + "gitea.stevedudenhoeffer.com/steve/majordomo/audio" "gitea.stevedudenhoeffer.com/steve/majordomo/llm" ) @@ -20,11 +20,11 @@ import ( // served by llama-swap (routed to a kokoro/chatterbox-style OpenAI-compatible // upstream). The id is passed through verbatim and selects which upstream // llama-swap loads. -func (p *Provider) SpeechModel(id string, opts ...majaudio.SpeechModelOption) (majaudio.SpeechModel, error) { - if p.baseURL == "" { - return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) +func (p *Provider) SpeechModel(id string, opts ...audio.SpeechModelOption) (audio.SpeechModel, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err } - _ = majaudio.ApplySpeechModelOptions(opts) + _ = audio.ApplySpeechModelOptions(opts) return &speechModel{p: p, id: id}, nil } @@ -44,11 +44,14 @@ type speechRequest struct { } // Speak implements audio.SpeechModel via POST {base}/v1/audio/speech. -func (m *speechModel) Speak(ctx context.Context, req majaudio.SpeechRequest, opts ...majaudio.SpeechOption) (*majaudio.SpeechResult, error) { +func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) { req = req.Apply(opts...) if strings.TrimSpace(req.Input) == "" { return nil, fmt.Errorf("%w: speech synthesis requires input text", llm.ErrUnsupported) } + if req.Speed < 0 { + return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed) + } wire := speechRequest{ Model: m.id, Input: req.Input, @@ -67,21 +70,17 @@ func (m *speechModel) Speak(ctx context.Context, req majaudio.SpeechRequest, opt if len(audioBytes) == 0 { return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech response contained no audio"} } - return &majaudio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil + return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil } // speechMIME resolves the result MIME type: the response Content-Type when it // is a concrete audio type, else a mapping from the requested format, else // audio/mpeg (the OpenAI endpoint's default container is mp3). func speechMIME(contentType, format string) string { - if mt, _, err := mime.ParseMediaType(contentType); err == nil { - if strings.HasPrefix(mt, "audio/") || strings.HasPrefix(mt, "video/") { - return mt - } + if mt, _, err := mime.ParseMediaType(contentType); err == nil && strings.HasPrefix(mt, "audio/") { + return mt } switch strings.ToLower(strings.TrimSpace(format)) { - case "", "mp3": - return "audio/mpeg" case "wav": return "audio/wav" case "opus": @@ -90,9 +89,7 @@ func speechMIME(contentType, format string) string { return "audio/aac" case "flac": return "audio/flac" - case "pcm": - return "audio/pcm" - default: + default: // "", "mp3", and anything unrecognized return "audio/mpeg" } } @@ -100,11 +97,11 @@ func speechMIME(contentType, format string) string { // TranscriptionModel implements audio.TranscriptionProvider, binding a // speech-to-text model served by llama-swap (routed to a whisper.cpp-style // upstream). -func (p *Provider) TranscriptionModel(id string, opts ...majaudio.TranscriptionModelOption) (majaudio.TranscriptionModel, error) { - if p.baseURL == "" { - return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) +func (p *Provider) TranscriptionModel(id string, opts ...audio.TranscriptionModelOption) (audio.TranscriptionModel, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err } - _ = majaudio.ApplyTranscriptionModelOptions(opts) + _ = audio.ApplyTranscriptionModelOptions(opts) return &transcriptionModel{p: p, id: id}, nil } @@ -116,7 +113,7 @@ type transcriptionModel struct { // Transcribe implements audio.TranscriptionModel via POST // {base}/v1/audio/transcriptions (multipart/form-data — llama-swap routes by // the `model` form field). -func (m *transcriptionModel) Transcribe(ctx context.Context, req majaudio.TranscriptionRequest, opts ...majaudio.TranscriptionOption) (*majaudio.TranscriptionResult, error) { +func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.TranscriptionRequest, opts ...audio.TranscriptionOption) (*audio.TranscriptionResult, error) { req = req.Apply(opts...) if len(req.Audio) == 0 { return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported) @@ -131,17 +128,22 @@ func (m *transcriptionModel) Transcribe(ctx context.Context, req majaudio.Transc if _, err := fw.Write(req.Audio); err != nil { return nil, fmt.Errorf("llama-swap: build transcription form: %w", err) } - fields := map[string]string{ - "model": m.id, - "language": req.Language, - "prompt": req.Prompt, - "response_format": "json", + // Required fields always go on the wire (an empty model id should fail + // loudly upstream, not silently vanish); optional ones only when set. + fields := []struct { + key, value string + required bool + }{ + {"model", m.id, true}, + {"response_format", "json", true}, + {"language", req.Language, false}, + {"prompt", req.Prompt, false}, } - for k, v := range fields { - if v == "" { + for _, f := range fields { + if !f.required && f.value == "" { continue } - if err := w.WriteField(k, v); err != nil { + if err := w.WriteField(f.key, f.value); err != nil { return nil, fmt.Errorf("llama-swap: build transcription form: %w", err) } } @@ -159,22 +161,31 @@ func (m *transcriptionModel) Transcribe(ctx context.Context, req majaudio.Transc if err := json.Unmarshal(raw, &out); err != nil { return nil, fmt.Errorf("llama-swap: decode transcription response: %w", err) } - return &majaudio.TranscriptionResult{Text: out.Text, Raw: json.RawMessage(raw)}, nil + return &audio.TranscriptionResult{Text: out.Text, Raw: json.RawMessage(raw)}, nil } -// transcriptionFilename picks the multipart filename hint: the caller's, else -// one derived from the MIME subtype ("audio.mp3"), else "audio". -func transcriptionFilename(req majaudio.TranscriptionRequest) string { - if req.Filename != "" { - return req.Filename +// transcriptionFilename picks the multipart filename hint: the caller's +// (sanitized — upload metadata is untrusted and CR/LF would inject multipart +// headers), else one derived from the MIME subtype ("audio.mp3"), else +// "audio". MIME parameters ("audio/ogg; codecs=opus") are stripped before +// matching. +func transcriptionFilename(req audio.TranscriptionRequest) string { + if name := sanitizeFilename(req.Filename); name != "" { + return name } - switch strings.ToLower(req.MIME) { + mt := strings.ToLower(strings.TrimSpace(req.MIME)) + if parsed, _, err := mime.ParseMediaType(mt); err == nil { + mt = parsed + } + switch mt { case "audio/mpeg", "audio/mp3": return "audio.mp3" case "audio/wav", "audio/x-wav", "audio/wave": return "audio.wav" - case "audio/ogg", "audio/opus": + case "audio/ogg": return "audio.ogg" + case "audio/opus": + return "audio.opus" case "audio/flac", "audio/x-flac": return "audio.flac" case "audio/mp4", "audio/m4a", "audio/x-m4a": @@ -186,6 +197,14 @@ func transcriptionFilename(req majaudio.TranscriptionRequest) string { } } +// sanitizeFilename strips characters that would corrupt or inject into the +// multipart Content-Disposition header. Quotes and backslashes are escaped +// by mime/multipart itself; CR/LF are not — they must go. +func sanitizeFilename(name string) string { + name = strings.NewReplacer("\r", "", "\n", "").Replace(name) + return strings.TrimSpace(name) +} + // ListVoices returns the voices a TTS model offers (GET // /v1/audio/voices?model=...). The decode is tolerant: upstreams answer with // either a bare string list or a list of {id|name} objects. @@ -221,7 +240,9 @@ func parseVoices(raw []byte) ([]string, error) { if err := json.Unmarshal(list, &names); err == nil { return names, nil } - // A failed decode above may have partially populated names — start fresh. + // Not a plain string list. The failed decode above may have partially + // populated names (Unmarshal appends zero values before erroring on an + // element type mismatch) — start fresh for the object shape. names = nil var objs []struct { ID string `json:"id"` @@ -247,20 +268,15 @@ func parseVoices(raw []byte) ([]string, error) { // doRaw performs a request to a llama-swap endpoint and returns the raw // response body and its Content-Type — the sibling of doJSON for endpoints // whose success payload is not JSON (audio bytes) or whose shape varies. -// contentType sets the request Content-Type when body is non-nil. +// contentType sets the request Content-Type when body is non-nil. A response +// larger than maxResponseBytes is an error, never a silent truncation. func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType string, body io.Reader) ([]byte, string, error) { - if p.baseURL == "" { - return nil, "", fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + if err := p.requireBaseURL(); err != nil { + return nil, "", err } - req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, body) + req, err := p.newRequest(ctx, method, path, contentType, body) if err != nil { - return nil, "", fmt.Errorf("llama-swap: build request: %w", err) - } - if body != nil && contentType != "" { - req.Header.Set("Content-Type", contentType) - } - if p.token != "" { - req.Header.Set("Authorization", "Bearer "+p.token) + return nil, "", err } resp, err := p.client.Do(req) if err != nil { @@ -270,9 +286,12 @@ func (p *Provider) doRaw(ctx context.Context, method, path, model, contentType s if resp.StatusCode/100 != 2 { return nil, "", p.apiError(resp, model) } - data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes)) + data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) if err != nil { return nil, "", fmt.Errorf("llama-swap: read response: %w", err) } + if len(data) > maxResponseBytes { + return nil, "", fmt.Errorf("llama-swap: response exceeds %d bytes", maxResponseBytes) + } return data, resp.Header.Get("Content-Type"), nil } diff --git a/provider/llamaswap/health.go b/provider/llamaswap/health.go index 50a679a..7c25253 100644 --- a/provider/llamaswap/health.go +++ b/provider/llamaswap/health.go @@ -5,6 +5,8 @@ import ( "fmt" "io" "net/http" + + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" ) // Health reports whether the llama-swap instance is reachable (GET @@ -13,16 +15,17 @@ import ( // request will take (a cold model swap can still block for minutes). Bound // the probe with a short context deadline; the client has no timeout by // design. +// +// Failure taxonomy matches the rest of the package: transport failures wrap +// the raw net error; a reachable-but-unhealthy status comes back as +// *llm.APIError, so callers can errors.As-distinguish the two. func (p *Provider) Health(ctx context.Context) error { - if p.baseURL == "" { - return fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + if err := p.requireBaseURL(); err != nil { + return err } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.baseURL+"/health", nil) + req, err := p.newRequest(ctx, http.MethodGet, "/health", "", nil) if err != nil { - return fmt.Errorf("llama-swap: build request: %w", err) - } - if p.token != "" { - req.Header.Set("Authorization", "Bearer "+p.token) + return err } resp, err := p.client.Do(req) if err != nil { @@ -31,15 +34,11 @@ func (p *Provider) Health(ctx context.Context) error { defer resp.Body.Close() _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<10)) if resp.StatusCode/100 != 2 { - return &apiHealthError{status: resp.StatusCode} + return &llm.APIError{ + Provider: p.name, + Status: resp.StatusCode, + Message: "health endpoint returned a non-2xx status", + } } return nil } - -// apiHealthError distinguishes "reachable but unhealthy" from transport -// failure without pulling in the llm error taxonomy for a probe. -type apiHealthError struct{ status int } - -func (e *apiHealthError) Error() string { - return fmt.Sprintf("llama-swap: health endpoint returned status %d", e.status) -} diff --git a/provider/llamaswap/image.go b/provider/llamaswap/image.go index c5edd19..e10af4c 100644 --- a/provider/llamaswap/image.go +++ b/provider/llamaswap/image.go @@ -16,8 +16,8 @@ import ( // served by llama-swap (routed to a stable-diffusion.cpp upstream). The id is // passed through verbatim and selects which upstream llama-swap loads. func (p *Provider) ImageModel(id string, opts ...imagegen.ModelOption) (imagegen.Model, error) { - if p.baseURL == "" { - return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + if err := p.requireBaseURL(); err != nil { + return nil, err } _ = imagegen.ApplyModelOptions(opts) return &imageModel{p: p, id: id}, nil @@ -53,32 +53,39 @@ type txt2imgResponse struct { Images []string `json:"images"` } +// sdWire validates the generation knobs shared by Generate and Edit and +// builds the common txt2img wire fields. verb labels validation errors. +func (m *imageModel) sdWire(verb, prompt, negativePrompt, sampler, size string, seed *int64, steps *int, cfgScale *float64, n int) (txt2imgRequest, error) { + if strings.TrimSpace(prompt) == "" { + return txt2imgRequest{}, fmt.Errorf("%w: image %s requires a prompt", llm.ErrUnsupported, verb) + } + if n < 0 { + return txt2imgRequest{}, fmt.Errorf("%w: image count N must be >= 0, got %d", llm.ErrUnsupported, n) + } + width, height, err := parseSize(size) + if err != nil { + return txt2imgRequest{}, fmt.Errorf("%w: %v", llm.ErrUnsupported, err) + } + return txt2imgRequest{ + Model: m.id, + Prompt: prompt, + NegativePrompt: negativePrompt, + Seed: seed, + Steps: steps, + CFGScale: cfgScale, + Width: width, + Height: height, + SampleMethod: sampler, + BatchCount: n, + }, nil +} + // Generate implements imagegen.Model via POST {base}/sdapi/v1/txt2img. func (m *imageModel) Generate(ctx context.Context, req imagegen.Request, opts ...imagegen.Option) (*imagegen.Result, error) { req = req.Apply(opts...) - if strings.TrimSpace(req.Prompt) == "" { - return nil, fmt.Errorf("%w: image generation requires a prompt", llm.ErrUnsupported) - } - if req.N < 0 { - return nil, fmt.Errorf("%w: image count N must be >= 0, got %d", llm.ErrUnsupported, req.N) - } - - width, height, err := parseSize(req.Size) + wire, err := m.sdWire("generation", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N) if err != nil { - return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err) - } - - wire := txt2imgRequest{ - Model: m.id, - Prompt: req.Prompt, - NegativePrompt: req.NegativePrompt, - Seed: req.Seed, - Steps: req.Steps, - CFGScale: req.CFGScale, - Width: width, - Height: height, - SampleMethod: req.Sampler, - BatchCount: req.N, + return nil, err } var resp txt2imgResponse @@ -126,37 +133,18 @@ type img2imgRequest struct { // Edit implements imagegen.Editor via POST {base}/sdapi/v1/img2img. func (m *imageModel) Edit(ctx context.Context, req imagegen.EditRequest, opts ...imagegen.EditOption) (*imagegen.Result, error) { req = req.Apply(opts...) - if strings.TrimSpace(req.Prompt) == "" { - return nil, fmt.Errorf("%w: image edit requires a prompt", llm.ErrUnsupported) - } if len(req.Init.Data) == 0 { return nil, fmt.Errorf("%w: image edit requires an init image", llm.ErrUnsupported) } - if req.N < 0 { - return nil, fmt.Errorf("%w: image count N must be >= 0, got %d", llm.ErrUnsupported, req.N) - } if req.Strength != nil && (*req.Strength < 0 || *req.Strength > 1) { return nil, fmt.Errorf("%w: edit strength must be in [0,1], got %g", llm.ErrUnsupported, *req.Strength) } - - width, height, err := parseSize(req.Size) + base, err := m.sdWire("edit", req.Prompt, req.NegativePrompt, req.Sampler, req.Size, req.Seed, req.Steps, req.CFGScale, req.N) if err != nil { - return nil, fmt.Errorf("%w: %v", llm.ErrUnsupported, err) + return nil, err } - wire := img2imgRequest{ - txt2imgRequest: txt2imgRequest{ - Model: m.id, - Prompt: req.Prompt, - NegativePrompt: req.NegativePrompt, - Seed: req.Seed, - Steps: req.Steps, - CFGScale: req.CFGScale, - Width: width, - Height: height, - SampleMethod: req.Sampler, - BatchCount: req.N, - }, + txt2imgRequest: base, InitImages: []string{base64.StdEncoding.EncodeToString(req.Init.Data)}, DenoisingStrength: req.Strength, } diff --git a/provider/llamaswap/llamaswap.go b/provider/llamaswap/llamaswap.go index f973cf1..c175a44 100644 --- a/provider/llamaswap/llamaswap.go +++ b/provider/llamaswap/llamaswap.go @@ -9,9 +9,11 @@ // package adds beyond a bare OpenAI-compat endpoint is the "tailored" surface: // // - llama-swap management endpoints exposed as concrete methods — ListModels -// (GET /v1/models), Running (GET /running), Unload (POST /api/models/unload) -// — which have no place on the canonical llm.Provider interface; -// - image generation via the imagegen interface (see image.go); and +// (GET /v1/models), Running (GET /running), Unload (POST /api/models/unload), +// ListVoices (GET /v1/audio/voices), Health (GET /health) — which have no +// place on the canonical llm.Provider interface; +// - image generation + editing via the imagegen interfaces (see image.go); +// - speech synthesis + transcription via the audio interfaces (see audio.go); and // - swap-aware defaults: the HTTP client carries NO timeout, because the // first request to an unloaded model blocks while llama-swap spawns the // upstream (its healthCheckTimeout is at least 15s). Bound a call with a @@ -100,8 +102,8 @@ func (p *Provider) BaseURL() string { return p.baseURL } // endpoint, delegating to provider/openai. The id is passed through verbatim // and selects which upstream llama-swap loads. func (p *Provider) Model(id string, opts ...llm.ModelOption) (llm.Model, error) { - if p.baseURL == "" { - return nil, fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + if err := p.requireBaseURL(); err != nil { + return nil, err } return p.chatProvider().Model(id, opts...) } @@ -178,7 +180,32 @@ func (p *Provider) Unload(ctx context.Context, model string) error { return p.doJSON(ctx, http.MethodPost, path, "", nil, nil) } -// --- shared HTTP helper for management + image endpoints --- +// --- shared HTTP helpers for management + image + audio endpoints --- + +// requireBaseURL is the shared guard for every entry point: construction +// never fails (see New), so a missing base URL surfaces here, at use time. +func (p *Provider) requireBaseURL() error { + if p.baseURL == "" { + return fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + } + return nil +} + +// newRequest builds an authenticated request relative to baseURL. +// contentType is applied only when a body is present. +func (p *Provider) newRequest(ctx context.Context, method, path, contentType string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, body) + if err != nil { + return nil, fmt.Errorf("llama-swap: build request: %w", err) + } + if body != nil && contentType != "" { + req.Header.Set("Content-Type", contentType) + } + if p.token != "" { + req.Header.Set("Authorization", "Bearer "+p.token) + } + return req, nil +} // doJSON performs a request to a llama-swap endpoint relative to baseURL, // optionally encoding body and decoding into out (either may be nil). model @@ -186,8 +213,8 @@ func (p *Provider) Unload(ctx context.Context, model string) error { // model-specific). Transport failures are wrapped raw so llm.Classify still // sees the underlying net error; non-2xx responses become *llm.APIError. func (p *Provider) doJSON(ctx context.Context, method, path, model string, body, out any) error { - if p.baseURL == "" { - return fmt.Errorf("llama-swap provider %q: no base URL configured (set one via WithBaseURL or an LLM_* env DSN)", p.name) + if err := p.requireBaseURL(); err != nil { + return err } var rdr io.Reader if body != nil { @@ -197,15 +224,9 @@ func (p *Provider) doJSON(ctx context.Context, method, path, model string, body, } rdr = bytes.NewReader(b) } - req, err := http.NewRequestWithContext(ctx, method, p.baseURL+path, rdr) + req, err := p.newRequest(ctx, method, path, "application/json", rdr) if err != nil { - return fmt.Errorf("llama-swap: build request: %w", err) - } - if body != nil { - req.Header.Set("Content-Type", "application/json") - } - if p.token != "" { - req.Header.Set("Authorization", "Bearer "+p.token) + return err } resp, err := p.client.Do(req) if err != nil {