feat: wave-3 audio surfaces — stems, SFX, speech enhance, voice clone, translate (ADR-0024) #18

Merged
steve merged 2 commits from feat/wave3-audio-surfaces into main 2026-07-16 23:24:42 +00:00
12 changed files with 1275 additions and 5 deletions
+28
View File
@@ -30,6 +30,17 @@ type SpeechRequest struct {
// Speed is the playback-rate multiplier; 0 = backend default (1.0). // Speed is the playback-rate multiplier; 0 = backend default (1.0).
Speed float64 Speed float64
// ReferenceAudio is a short voice sample for zero-shot voice cloning
// (chatterbox style): when set, the model speaks Input in the sampled
// voice instead of a named Voice. nil = normal synthesis. Backends
// without cloning support must reject a reference-carrying request
// rather than silently ignoring it.
ReferenceAudio []byte
// ReferenceMIME is the reference audio's MIME type (e.g. "audio/wav");
// "" = let the backend sniff it.
ReferenceMIME string
} }
// SpeechResult is the canonical synthesis result: raw audio bytes plus the // SpeechResult is the canonical synthesis result: raw audio bytes plus the
@@ -59,6 +70,11 @@ func WithFormat(f string) SpeechOption { return func(r *SpeechRequest) { r.Forma
// WithSpeed sets the playback-rate multiplier. // WithSpeed sets the playback-rate multiplier.
func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } } func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } }
// WithReferenceAudio provides a voice sample for zero-shot voice cloning.
func WithReferenceAudio(data []byte, mime string) SpeechOption {
return func(r *SpeechRequest) { r.ReferenceAudio, r.ReferenceMIME = data, mime }
}
// Apply returns a copy of the request with all options applied. Providers // Apply returns a copy of the request with all options applied. Providers
// call this once at the top of Speak. // call this once at the top of Speak.
func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest { func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest {
@@ -121,6 +137,13 @@ type TranscriptionRequest struct {
// Prompt is optional context or vocabulary to bias decoding; "" = none. // Prompt is optional context or vocabulary to bias decoding; "" = none.
Prompt string Prompt string
// Translate requests an English translation of the speech instead of a
// same-language transcript (whisper's translate task). When set and no
// Language is given, providers must force source-language auto-detection
// — a backend whose default language is "en" would otherwise skip
// translation entirely.
Translate bool
} }
// TranscriptionResult is the canonical transcription result. // TranscriptionResult is the canonical transcription result.
@@ -145,6 +168,11 @@ func WithPrompt(p string) TranscriptionOption {
return func(r *TranscriptionRequest) { r.Prompt = p } return func(r *TranscriptionRequest) { r.Prompt = p }
} }
// WithTranslate requests an English translation instead of a transcript.
func WithTranslate() TranscriptionOption {
return func(r *TranscriptionRequest) { r.Translate = true }
}
// Apply returns a copy of the request with all options applied. // Apply returns a copy of the request with all options applied.
func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest { func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest {
for _, opt := range opts { for _, opt := range opts {
+66
View File
@@ -0,0 +1,66 @@
package audio
import "context"
// EnhancementRequest asks a speech-enhancement backend (DeepFilterNet style)
// to denoise a recording. Audio is carried as bytes (never a URL), mirroring
// TranscriptionRequest (ADR-0024).
type EnhancementRequest struct {
// Audio is the encoded audio to enhance.
Audio []byte
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
// sniff it.
MIME string
// Filename is the multipart filename hint some backends key their format
// detection on; "" derives one from MIME or falls back to "audio".
Filename string
}
// EnhancementOption mutates an EnhancementRequest before it is sent.
// Reserved for future request settings (the reference backend takes no
// parameters).
type EnhancementOption func(*EnhancementRequest)
// Apply returns a copy of the request with all options applied.
func (r EnhancementRequest) Apply(opts ...EnhancementOption) EnhancementRequest {
for _, opt := range opts {
opt(&r)
}
return r
}
// SpeechEnhancer denoises speech recordings. The result reuses SpeechResult
// (audio bytes + MIME + Raw) — enhancement is audio-in/audio-out exactly
// like synthesis.
type SpeechEnhancer interface {
// Enhance returns the denoised audio.
Enhance(ctx context.Context, req EnhancementRequest, opts ...EnhancementOption) (*SpeechResult, error)
}
// SpeechEnhancerModelOption configures a SpeechEnhancer at construction time.
// Reserved for future per-model settings.
type SpeechEnhancerModelOption func(*SpeechEnhancerModelConfig)
Review

**Repeated ModelOption/ModelConfig boilerplate across audio/ files could be consolidated or better documented

maintainability · flagged by 1 model

  • audio/enhance.go:44 / audio/stems.go:91 — The *ModelOption / *ModelConfig boilerplate pattern is repeated verbatim across 5+ files in audio/ (audio.go:96, audio.go:192, diarize.go:91, enhance.go:44, stems.go:91). This is consistent with existing code, but the comment "Reserved for future per-model settings" appears in every file without any actual settings ever being added. If this is truly placeholder infrastructure, consider consolidating into a single shared pattern…

🪰 Gadfly · advisory

⚪ **Repeated *ModelOption/*ModelConfig boilerplate across audio/ files could be consolidated or better documented** _maintainability · flagged by 1 model_ - **`audio/enhance.go:44` / `audio/stems.go:91`** — The `*ModelOption` / `*ModelConfig` boilerplate pattern is repeated verbatim across 5+ files in `audio/` (`audio.go:96`, `audio.go:192`, `diarize.go:91`, `enhance.go:44`, `stems.go:91`). This is consistent with existing code, but the comment "Reserved for future per-model settings" appears in every file without any actual settings ever being added. If this is truly placeholder infrastructure, consider consolidating into a single shared pattern… <sub>🪰 Gadfly · advisory</sub>
// SpeechEnhancerModelConfig carries per-model construction settings.
type SpeechEnhancerModelConfig struct{}
// ApplySpeechEnhancerModelOptions folds options into a config.
func ApplySpeechEnhancerModelOptions(opts []SpeechEnhancerModelOption) SpeechEnhancerModelConfig {
var cfg SpeechEnhancerModelConfig
for _, opt := range opts {
opt(&cfg)
}
return cfg
}
// SpeechEnhancementProvider mints SpeechEnhancers bound to one backend.
type SpeechEnhancementProvider interface {
// Name is the registry identifier for the provider.
Name() string
// SpeechEnhancerModel returns a SpeechEnhancer bound to the given id
// (passed through to the backend verbatim; no catalog validation).
SpeechEnhancerModel(id string, opts ...SpeechEnhancerModelOption) (SpeechEnhancer, error)
}
+113
View File
@@ -0,0 +1,113 @@
package audio
import "context"
// StemSeparationRequest asks a source-separation backend (Demucs style) to
// split a mix into stems. Audio is carried as bytes (never a URL), mirroring
// TranscriptionRequest. Zero values mean "backend default" (ADR-0024).
type StemSeparationRequest struct {
// Audio is the encoded mix to separate.
Audio []byte
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
// sniff it.
MIME string
// Filename is the multipart filename hint some backends key their format
// detection on; "" derives one from MIME or falls back to "audio".
Filename string
// Mode selects the split: "two" (vocals + accompaniment) or "four"
// (vocals/drums/bass/other); "" = backend default (four).
Mode string
// Model selects the separator's internal network where the backend
// offers several (Demucs: "htdemucs", "htdemucs_ft"); "" = backend
// default. This is NOT the provider model id — that is fixed when the
// StemSeparator is minted (mirrors BackgroundRemovalRequest.Net).
Model string
// Format is the per-stem audio container ("mp3" or "wav");
// "" = backend default.
Format string
}
// Stem is one separated source.
type Stem struct {
// Name is the stem's name ("vocals", "drums", "bass", "other",
// "no_vocals", ...), taken from the backend's own labelling.
Name string
// Audio is the encoded stem.
Audio []byte
// MIME is the stem's audio MIME type, e.g. "audio/mpeg".
MIME string
}
// StemSeparationResult is the canonical separation result.
type StemSeparationResult struct {
// Stems are the separated sources, in the order the backend returned
// them.
Stems []Stem
}
// StemSeparationOption mutates a StemSeparationRequest before it is sent.
type StemSeparationOption func(*StemSeparationRequest)
// WithStemMode selects the split ("two" or "four").
func WithStemMode(m string) StemSeparationOption {
return func(r *StemSeparationRequest) { r.Mode = m }
}
// WithStemModel selects the separator's internal network.
func WithStemModel(m string) StemSeparationOption {
return func(r *StemSeparationRequest) { r.Model = m }
}
// WithStemFormat sets the per-stem audio container ("mp3", "wav").
func WithStemFormat(f string) StemSeparationOption {
return func(r *StemSeparationRequest) { r.Format = f }
}
// Apply returns a copy of the request with all options applied.
func (r StemSeparationRequest) Apply(opts ...StemSeparationOption) StemSeparationRequest {
for _, opt := range opts {
opt(&r)
}
return r
}
// StemSeparator splits a mix into stems.
type StemSeparator interface {
// SeparateStems returns the separated sources. Separation is CPU-bound
// and slow (minutes for a full song); bound the call with a context
// deadline.
SeparateStems(ctx context.Context, req StemSeparationRequest, opts ...StemSeparationOption) (*StemSeparationResult, error)
}
// StemSeparatorModelOption configures a StemSeparator at construction time.
// Reserved for future per-model settings.
type StemSeparatorModelOption func(*StemSeparatorModelConfig)
// StemSeparatorModelConfig carries per-model construction settings.
type StemSeparatorModelConfig struct{}
// ApplyStemSeparatorModelOptions folds options into a config.
func ApplyStemSeparatorModelOptions(opts []StemSeparatorModelOption) StemSeparatorModelConfig {
var cfg StemSeparatorModelConfig
for _, opt := range opts {
opt(&cfg)
}
return cfg
}
// StemSeparationProvider mints StemSeparators bound to one backend.
type StemSeparationProvider interface {
// Name is the registry identifier for the provider.
Name() string
// StemSeparatorModel returns a StemSeparator bound to the given id
// (passed through to the backend verbatim; no catalog validation).
StemSeparatorModel(id string, opts ...StemSeparatorModelOption) (StemSeparator, error)
}
+65
View File
@@ -0,0 +1,65 @@
# ADR-0024: Wave-3 audio surfaces (stems, SFX, speech enhance, voice clone, translate)
Status: Accepted (2026-07-16)
## Context
The llama-swap host is gaining wave-3 audio capabilities (spec: mort
docs/specs/2026-07-16-llamaswap-wave3.md): Demucs stem separation and
DeepFilterNet speech enhancement (both on the CPU `audioutils` shim), Stable
Audio Open sound effects (`sfxgen`), plus two upgrades to existing models —
chatterbox's stateless voice-clone route and whisper.cpp's per-request
translate flag (both verified against the live images 2026-07-16).
## Decision
1. **`audio.StemSeparator` / `StemSeparationProvider`** —
`StemSeparationRequest{Audio, MIME, Filename, Mode, Model, Format}`
`StemSeparationResult{Stems []Stem{Name, Audio, MIME}}`.
- `Mode` is the caller-facing split: `"two"` (vocals + accompaniment,
sent as Demucs' `two_stems=vocals`) or `"four"`; `""` = backend
default. `Model` selects the Demucs variant (`htdemucs`/`htdemucs_ft`),
mirroring `BackgroundRemovalRequest.Net`.
- **The wire format is a ZIP** (`POST /upstream/<id>/v1/stems`): four WAV
stems would blow any JSON-of-base64 budget. Entry name → stem name,
extension → MIME; entries may sit under a per-model directory. Unpacking
is bounded per entry (zip-bomb guard) and a non-zip 2xx body fails loud.
2. **`SFXModel` reuses `musicgen`** — a sound effect is a short audio clip
from a text prompt; only the provider method differs. The sfxgen route
(`POST /upstream/<id>/v1/sfx`, JSON `{prompt, seconds, steps?, cfg_scale?,
seed?}`) is SYNCHRONOUS (WAV body), unlike ACE-Step's job queue.
`musicgen.Request` gains `CFGScale *float64` for it; lyrics and non-wav
formats are rejected (the model can't honor them). The ~11s model ceiling
is the backend's to enforce.
3. **`audio.SpeechEnhancer` / `SpeechEnhancementProvider`** —
`EnhancementRequest{Audio, MIME, Filename}`
`POST /upstream/<id>/v1/enhance` → WAV. The result reuses `SpeechResult`;
audio-in/audio-out needs no new result type.
4. **Voice cloning is a `SpeechRequest` field, not a new surface**
`ReferenceAudio []byte` + `ReferenceMIME`. When set, the llamaswap
speech model switches from JSON `/v1/audio/speech` to multipart
`POST /upstream/<id>/v1/audio/speech/upload` (fields `input` +
`voice_file`; verified live: stateless per-request cloning, no voice
library). Voice/format/speed still ride when set; the clone route's MIME
fallback is wav (not the JSON route's mp3). Backends without cloning must
reject a reference-carrying request rather than silently ignore it.
5. **Translation is a `TranscriptionRequest` bool**`Translate` maps to
whisper.cpp's `translate=true` form field (server.cpp:534). Because that
server's language DEFAULT is `en` (which silently skips translation), the
provider forces `language=auto` when translating without an explicit
language hint; an explicit hint wins.
6. **Binary success bodies are validated before wrapping** (ADR-0020 rule):
sfx/enhance require positive evidence of audio-ness (declared audio/*
Content-Type or sniffed RIFF/WAVE), stems require a parseable zip with
at least one stem.
## Consequences
- `audio` grows from three surfaces to five; the SFX surface adds zero new
types (musicgen reuse) — one format→MIME table and one multipart builder
keep serving every audio endpoint.
- The clone-route switch means one `SpeechModel` can answer over two wire
shapes; tests pin both routes so a regression can't silently drop cloning.
- `two_stems` is hardwired to vocals: "isolate X vs the rest" for other
sources is a backend capability not exposed in v1 (add a field when a
caller needs it, not before).
+8
View File
@@ -40,6 +40,11 @@ type Request struct {
// Steps is the number of inference steps; nil = backend default. // Steps is the number of inference steps; nil = backend default.
Steps *int Steps *int
// CFGScale is the classifier-free-guidance scale; nil = backend default.
// Architecture-sensitive, so prefer leaving it nil unless the caller
// knows the target model. Backends without the knob ignore it.
CFGScale *float64
// Seed fixes the RNG seed for reproducible output; nil = backend // Seed fixes the RNG seed for reproducible output; nil = backend
// default (random). // default (random).
Seed *int64 Seed *int64
@@ -73,6 +78,9 @@ func WithFormat(f string) Option { return func(r *Request) { r.Format = f } }
// WithSteps overrides the number of inference steps. // WithSteps overrides the number of inference steps.
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } } func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }
// WithCFGScale overrides the classifier-free-guidance scale.
func WithCFGScale(s float64) Option { return func(r *Request) { r.CFGScale = &s } }
// WithSeed fixes the RNG seed. // WithSeed fixes the RNG seed.
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } } func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }
+93 -5
View File
@@ -9,6 +9,7 @@ import (
"mime" "mime"
"net/http" "net/http"
"net/url" "net/url"
"strconv"
"strings" "strings"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio" "gitea.stevedudenhoeffer.com/steve/majordomo/audio"
@@ -42,7 +43,11 @@ type speechRequest struct {
Speed float64 `json:"speed,omitempty"` Speed float64 `json:"speed,omitempty"`
} }
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech. // Speak implements audio.SpeechModel via POST {base}/v1/audio/speech, or —
// when the request carries reference audio for voice cloning — via multipart
// POST {base}/upstream/<id>/v1/audio/speech/upload (the chatterbox clone
// route; verified live 2026-07-16: stateless per-request cloning, fields
// `input` + `voice_file`).
func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) { func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) {
req = req.Apply(opts...) req = req.Apply(opts...)
if strings.TrimSpace(req.Input) == "" { if strings.TrimSpace(req.Input) == "" {
@@ -51,6 +56,9 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
if req.Speed < 0 { if req.Speed < 0 {
return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed) return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed)
} }
if len(req.ReferenceAudio) > 0 {
return m.speakWithReference(ctx, req)
}
wire := speechRequest{ wire := speechRequest{
Model: m.id, Model: m.id,
Input: req.Input, Input: req.Input,
@@ -72,6 +80,48 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil
} }
// speakWithReference performs zero-shot voice cloning via the upstream
// passthrough clone route. The reference sample rides as the `voice_file`
// part and the text as an `input` field; voice/format/speed still go on the
// wire when set (upstreams ignore fields they don't understand). The result
// MIME comes from the response header or content sniffing (audioResultMIME,
// same validation as the sfx/enhance surfaces) — a JSON soft error or an
// HTML proxy page must never be wrapped up as audio bytes.
func (m *speechModel) speakWithReference(ctx context.Context, req audio.SpeechRequest) (*audio.SpeechResult, error) {
upPath, err := upstreamPath(m.id, "/v1/audio/speech/upload")
Review

Local var named path here vs upPath in stems.go/enhance.go — minor naming drift

maintainability · flagged by 1 model

  • Minor local-naming drift for the upstream path variable. stems.go:57/enhance.go name it upPath (deliberately, to dodge the path package import in stems.go), while speakWithReference (provider/llamaswap/audio.go:91) names it path (audio.go doesn't import the path package, so both compile). Purely cosmetic consistency, not a bug. Trivial.

🪰 Gadfly · advisory

⚪ **Local var named `path` here vs `upPath` in stems.go/enhance.go — minor naming drift** _maintainability · flagged by 1 model_ - **Minor local-naming drift for the upstream path variable.** `stems.go:57`/`enhance.go` name it `upPath` (deliberately, to dodge the `path` package import in `stems.go`), while `speakWithReference` (`provider/llamaswap/audio.go:91`) names it `path` (audio.go doesn't import the `path` package, so both compile). Purely cosmetic consistency, not a bug. Trivial. <sub>🪰 Gadfly · advisory</sub>
if err != nil {
return nil, err
}
speed := ""
if req.Speed != 0 {
speed = strconv.FormatFloat(req.Speed, 'g', -1, 64)
}
body, formType, err := buildMultipart("build speech clone form",
filePart{field: "voice_file", filename: transcriptionFilename("", req.ReferenceMIME), data: req.ReferenceAudio},
[]formField{
{"input", req.Input, true},
{"voice", req.Voice, false},
{"response_format", req.Format, false},
{"speed", speed, false},
})
if err != nil {
return nil, err
}
audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, upPath, m.id, formType, body, maxAudioResponseBytes)
Review

🔴 speech clone route does not validate response is audio before wrapping in SpeechResult

error-handling, security · flagged by 2 models

  • provider/llamaswap/audio.go:113-119speakWithReference resolves a MIME type and returns the raw response body as a successful SpeechResult without ever checking that the bytes are actually audio. The PR's own ADR-0024 point 6 states "Binary success bodies are validated before wrapping," and the new sfx.go and enhance.go both enforce this with audioResultMIME; the clone route does not. A misconfigured upstream returning an HTML error page or JSON soft-error will be returned to…

🪰 Gadfly · advisory

🔴 **speech clone route does not validate response is audio before wrapping in SpeechResult** _error-handling, security · flagged by 2 models_ - **`provider/llamaswap/audio.go:113-119`** — `speakWithReference` resolves a MIME type and returns the raw response body as a successful `SpeechResult` without ever checking that the bytes are actually audio. The PR's own ADR-0024 point 6 states "Binary success bodies are validated before wrapping," and the new `sfx.go` and `enhance.go` both enforce this with `audioResultMIME`; the clone route does not. A misconfigured upstream returning an HTML error page or JSON soft-error will be returned to… <sub>🪰 Gadfly · advisory</sub>
if err != nil {
return nil, err
}
if len(audioBytes) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech clone response contained no audio"}
}
mimeType := audioResultMIME(contentType, audioBytes)
Review

🟠 speakWithReference lacks audio content sniffing validation present in enhance/sfx endpoints

error-handling, security · flagged by 2 models

  • speakWithReference lacks audio-content validation that the other new binary endpoints enforce (provider/llamaswap/audio.go:117-123). The enhance and sfx endpoints both use audioResultMIME, which falls back to http.DetectContentType sniffing and returns "" (→ error) when the response body is not recognizably audio. speakWithReference has no such sniffing fallback: if the upstream returns a non-audio Content-Type (e.g. text/html from a proxy error page) and req.Format i…

🪰 Gadfly · advisory

🟠 **speakWithReference lacks audio content sniffing validation present in enhance/sfx endpoints** _error-handling, security · flagged by 2 models_ - **`speakWithReference` lacks audio-content validation that the other new binary endpoints enforce** (`provider/llamaswap/audio.go:117-123`). The `enhance` and `sfx` endpoints both use `audioResultMIME`, which falls back to `http.DetectContentType` sniffing and returns `""` (→ error) when the response body is not recognizably audio. `speakWithReference` has no such sniffing fallback: if the upstream returns a non-audio `Content-Type` (e.g. `text/html` from a proxy error page) and `req.Format` i… <sub>🪰 Gadfly · advisory</sub>
if mimeType == "" {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("speech clone response is not audio (Content-Type %q): %s", contentType, truncateForError(audioBytes))}
}
return &audio.SpeechResult{Audio: audioBytes, MIME: mimeType}, nil
}
// speechMIME resolves the result MIME type: the response Content-Type when it // speechMIME resolves the result MIME type: the response Content-Type when it
// is a concrete audio type, else a mapping from the requested format, else // is a concrete audio type, else a mapping from the requested format, else
// audio/mpeg (the OpenAI endpoint's default container is mp3). // audio/mpeg (the OpenAI endpoint's default container is mp3).
@@ -93,6 +143,28 @@ func speechMIME(contentType, format string) string {
} }
} }
// audioResultMIME resolves an audio body's MIME type: the response
// Content-Type when it is a concrete audio type, else content sniffing
// (RIFF/WAVE, Ogg and friends), else "" — the caller treats undetectable as
// an upstream error, mirroring videoMIME. Sniffed "audio/wave" is normalized
// to the conventional "audio/wav", and Ogg's container type
// "application/ogg" to "audio/ogg". Shared by the clone, sfx, and enhance
// surfaces, which all answer raw audio bodies.
func audioResultMIME(contentType string, data []byte) string {
if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
return mt
}
switch mt := http.DetectContentType(data); {
case mt == "audio/wave":
return "audio/wav"
case mt == "application/ogg":
return "audio/ogg"
case strings.HasPrefix(mt, "audio/"):
return mt
}
return ""
}
// TranscriptionModel implements audio.TranscriptionProvider, binding a // TranscriptionModel implements audio.TranscriptionProvider, binding a
// speech-to-text model served by llama-swap (routed to a whisper.cpp-style // speech-to-text model served by llama-swap (routed to a whisper.cpp-style
// upstream). // upstream).
@@ -118,13 +190,26 @@ func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.Transcrip
return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported) return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported)
} }
// Translation is a per-request bool form field on whisper.cpp's server
// (server.cpp:534, verified 2026-07-16). Its language DEFAULT is "en",
// which would make translate a no-op — so an explicit language hint
// wins, but an unset one is forced to auto-detection.
translate := ""
language := req.Language
if req.Translate {
translate = "true"
if language == "" {
language = "auto"
}
}
buf, formType, err := buildMultipart("build transcription form", buf, formType, err := buildMultipart("build transcription form",
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio}, filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
[]formField{ []formField{
{"model", m.id, true}, {"model", m.id, true},
{"response_format", "json", true}, {"response_format", "json", true},
{"language", req.Language, false}, {"language", language, false},
{"prompt", req.Prompt, false}, {"prompt", req.Prompt, false},
{"translate", translate, false},
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@@ -177,10 +262,13 @@ func transcriptionFilename(filename, mimeType string) string {
} }
// sanitizeFilename strips characters that would corrupt or inject into the // sanitizeFilename strips characters that would corrupt or inject into the
// multipart Content-Disposition header. Quotes and backslashes are escaped // multipart Content-Disposition header, or smuggle directory structure to
// by mime/multipart itself; CR/LF are not — they must go. // the receiving side. Quotes and backslashes are escaped by mime/multipart
// itself, but a file-writing shim decodes them right back — so CR/LF/NUL
// and both path separators are dropped: an upload-metadata filename must
// never traverse ("../x", "a/b", "C:\x").
func sanitizeFilename(name string) string { func sanitizeFilename(name string) string {
name = strings.NewReplacer("\r", "", "\n", "").Replace(name) name = strings.NewReplacer("\r", "", "\n", "", "\x00", "", "/", "", "\\", "").Replace(name)
return strings.TrimSpace(name) return strings.TrimSpace(name)
} }
+215
View File
@@ -0,0 +1,215 @@
package llamaswap
import (
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"testing"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)
func TestSpeakWithReferenceUsesCloneRoute(t *testing.T) {
var gotPath, gotInput, gotVoice, gotFilename string
var gotRef []byte
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotInput = r.FormValue("input")
gotVoice = r.FormValue("voice")
f, hdr, err := r.FormFile("voice_file")
if err != nil {
t.Fatalf("voice_file part: %v", err)
}
defer f.Close()
gotRef, _ = io.ReadAll(f)
gotFilename = hdr.Filename
Review

🟡 Test helper wavFixture() defined in separate test file creates opaque cross-file dependency

maintainability · flagged by 1 model

  • provider/llamaswap/clone_translate_test.go:31 — Test helper wavFixture() is called but defined in a separate test file (stems_sfx_enhance_test.go:174). While legal Go (same package), this creates an opaque cross-file dependency that makes tests harder to navigate and maintain. Fix: Move wavFixture() to a shared test helper file (e.g., provider/llamaswap/testdata_test.go or provider/llamaswap/helpers_test.go) or duplicate it locally if it's trivial.

🪰 Gadfly · advisory

🟡 **Test helper wavFixture() defined in separate test file creates opaque cross-file dependency** _maintainability · flagged by 1 model_ - **`provider/llamaswap/clone_translate_test.go:31`** — Test helper `wavFixture()` is called but defined in a separate test file (`stems_sfx_enhance_test.go:174`). While legal Go (same package), this creates an opaque cross-file dependency that makes tests harder to navigate and maintain. **Fix:** Move `wavFixture()` to a shared test helper file (e.g., `provider/llamaswap/testdata_test.go` or `provider/llamaswap/helpers_test.go`) or duplicate it locally if it's trivial. <sub>🪰 Gadfly · advisory</sub>
w.Header().Set("Content-Type", "audio/wav")
_, _ = w.Write(wavFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SpeechModel("chatterbox")
res, err := sm.Speak(context.Background(),
audio.SpeechRequest{Input: "hello in my voice", Voice: "narrator"},
audio.WithReferenceAudio([]byte("REFWAV"), "audio/wav"))
if err != nil {
t.Fatalf("Speak: %v", err)
}
if gotPath != "/upstream/chatterbox/v1/audio/speech/upload" {
t.Errorf("path = %q, want clone route", gotPath)
}
if gotInput != "hello in my voice" || gotVoice != "narrator" {
t.Errorf("input/voice = %q/%q", gotInput, gotVoice)
}
if string(gotRef) != "REFWAV" || gotFilename != "audio.wav" {
t.Errorf("ref = %q name = %q", gotRef, gotFilename)
}
if res.MIME != "audio/wav" || len(res.Audio) == 0 {
t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio))
}
}
func TestSpeakWithReferenceSniffsHeaderlessWav(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
for _, k := range []string{"voice", "response_format", "speed"} {
if v, ok := r.MultipartForm.Value[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
w.Header()["Content-Type"] = nil // no declared type at all
_, _ = w.Write(wavFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SpeechModel("chatterbox")
res, err := sm.Speak(context.Background(),
audio.SpeechRequest{Input: "hi"},
audio.WithReferenceAudio([]byte("REF"), ""))
if err != nil {
t.Fatalf("Speak: %v", err)
}
// Headerless RIFF sniffs audio/wave, normalized to the conventional wav.
if res.MIME != "audio/wav" {
t.Errorf("MIME = %q, want sniffed audio/wav", res.MIME)
}
}
func TestSpeakWithReferenceRejectsNonAudioResponse(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// A FastAPI-style 2xx soft error must not be wrapped up as audio.
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"detail":"reference audio too short"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SpeechModel("chatterbox")
_, err := sm.Speak(context.Background(),
audio.SpeechRequest{Input: "hi"},
audio.WithReferenceAudio([]byte("REF"), "audio/wav"))
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for non-audio clone body", err)
}
}
func TestSanitizeFilenameStripsPathAndControlBytes(t *testing.T) {
for in, want := range map[string]string{
"song.mp3": "song.mp3",
"../../etc/passwd": "....etcpasswd",
"a\r\nContent-Type: evil": "aContent-Type: evil",
"..\\..\\boot.ini": "....boot.ini",
"nul\x00byte.wav": "nulbyte.wav",
" / ": "",
} {
if got := sanitizeFilename(in); got != want {
t.Errorf("sanitizeFilename(%q) = %q, want %q", in, got, want)
}
}
}
func TestSpeakWithoutReferenceKeepsJSONRoute(t *testing.T) {
var gotPath, gotContentType string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
gotContentType = r.Header.Get("Content-Type")
_, _ = w.Write([]byte("MP3"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SpeechModel("chatterbox")
if _, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: "hi"}); err != nil {
t.Fatalf("Speak: %v", err)
}
if gotPath != "/v1/audio/speech" || gotContentType != "application/json" {
t.Errorf("path/content-type = %q/%q, want JSON route", gotPath, gotContentType)
}
}
func TestTranscribeTranslateForcesAutoLanguage(t *testing.T) {
var gotTranslate, gotLanguage string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotTranslate = r.FormValue("translate")
gotLanguage = r.FormValue("language")
_, _ = w.Write([]byte(`{"text":"hello"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
res, err := tm.Transcribe(context.Background(),
audio.TranscriptionRequest{Audio: []byte("AUDIO")},
audio.WithTranslate())
if err != nil {
t.Fatalf("Transcribe: %v", err)
}
// whisper.cpp's server default language is "en", which would skip
// translation — translate must force auto-detection when no explicit
// language hint was given.
if gotTranslate != "true" || gotLanguage != "auto" {
t.Errorf("translate/language = %q/%q, want true/auto", gotTranslate, gotLanguage)
}
if res.Text != "hello" {
t.Errorf("text = %q", res.Text)
}
}
func TestTranscribeTranslateKeepsExplicitLanguage(t *testing.T) {
var gotTranslate, gotLanguage string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotTranslate = r.FormValue("translate")
gotLanguage = r.FormValue("language")
_, _ = w.Write([]byte(`{"text":"hello"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
if _, err := tm.Transcribe(context.Background(),
audio.TranscriptionRequest{Audio: []byte("AUDIO"), Language: "de", Translate: true}); err != nil {
t.Fatalf("Transcribe: %v", err)
}
if gotTranslate != "true" || gotLanguage != "de" {
t.Errorf("translate/language = %q/%q, want true/de", gotTranslate, gotLanguage)
}
}
func TestTranscribeWithoutTranslateOmitsFields(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
for _, k := range []string{"translate", "language"} {
if v, ok := r.MultipartForm.Value[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
_, _ = w.Write([]byte(`{"text":"hi"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
if _, err := tm.Transcribe(context.Background(),
audio.TranscriptionRequest{Audio: []byte("AUDIO")}); err != nil {
t.Fatalf("Transcribe: %v", err)
}
}
+61
View File
@@ -0,0 +1,61 @@
// enhance.go implements audio.SpeechEnhancementProvider against a
// DeepFilterNet shim (audioutils) reached through llama-swap's /upstream
// passthrough (ADR-0024):
//
// POST /upstream/<id>/v1/enhance multipart file -> WAV
package llamaswap
import (
"context"
"fmt"
"net/http"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)
// SpeechEnhancerModel implements audio.SpeechEnhancementProvider. The id
// selects which upstream llama-swap loads.
func (p *Provider) SpeechEnhancerModel(id string, opts ...audio.SpeechEnhancerModelOption) (audio.SpeechEnhancer, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = audio.ApplySpeechEnhancerModelOptions(opts)
return &speechEnhancerModel{p: p, id: id}, nil
}
type speechEnhancerModel struct {
p *Provider
id string
}
// Enhance implements audio.SpeechEnhancer. The endpoint always answers WAV.
func (m *speechEnhancerModel) Enhance(ctx context.Context, req audio.EnhancementRequest, opts ...audio.EnhancementOption) (*audio.SpeechResult, error) {
req = req.Apply(opts...)
if len(req.Audio) == 0 {
return nil, fmt.Errorf("%w: speech enhancement requires audio bytes", llm.ErrUnsupported)
}
path, err := upstreamPath(m.id, "/v1/enhance")
if err != nil {
return nil, err
}
body, contentType, err := buildMultipart("build enhance form",
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
nil)
if err != nil {
return nil, err
}
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxAudioResponseBytes)
Review

🟡 Enhance uses 64 MB JSON cap (maxResponseBytes) for WAV responses that can exceed it for recordings >10 min

correctness · flagged by 1 model

🪰 Gadfly · advisory

🟡 **Enhance uses 64 MB JSON cap (maxResponseBytes) for WAV responses that can exceed it for recordings >10 min** _correctness · flagged by 1 model_ <sub>🪰 Gadfly · advisory</sub>
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "enhance response contained no audio"}
}
mimeType := audioResultMIME(respType, raw)
Review

Shared audioResultMIME defined in sfx.go but used by enhance.go — poor discoverability

maintainability · flagged by 1 model

  • audioResultMIME lives in sfx.go but is also consumed by enhance.go (provider/llamaswap/enhance.go:55). It's a shared audio helper whose home file is one of its two callers, which hurts discoverability (a reader in enhance.go has to know to look in sfx.go). Since it's provider-wide, it reads more naturally alongside mimeFromContentType in llamaswap.go. Trivial, placement-only.

🪰 Gadfly · advisory

⚪ **Shared audioResultMIME defined in sfx.go but used by enhance.go — poor discoverability** _maintainability · flagged by 1 model_ - **`audioResultMIME` lives in `sfx.go` but is also consumed by `enhance.go` (`provider/llamaswap/enhance.go:55`).** It's a shared audio helper whose home file is one of its two callers, which hurts discoverability (a reader in `enhance.go` has to know to look in `sfx.go`). Since it's provider-wide, it reads more naturally alongside `mimeFromContentType` in `llamaswap.go`. Trivial, placement-only. <sub>🪰 Gadfly · advisory</sub>
if mimeType == "" {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("enhance response is not audio (Content-Type %q): %s", respType, truncateForError(raw))}
}
return &audio.SpeechResult{Audio: raw, MIME: mimeType}, nil
}
+6
View File
@@ -53,6 +53,12 @@ const maxResponseBytes = 64 << 20
// can't allocate without limit. // can't allocate without limit.
const maxVideoResponseBytes = 512 << 20 const maxVideoResponseBytes = 512 << 20
// maxAudioResponseBytes caps bodies that ARE a single encoded audio clip
// (voice clone, speech enhancement, sfx): a long uncompressed WAV
// legitimately passes the 64MB JSON cap. Still bounded so a buggy upstream
// can't allocate without limit.
const maxAudioResponseBytes = 256 << 20
// Provider is a llama-swap client. It satisfies llm.Provider (chat, delegated // Provider is a llama-swap client. It satisfies llm.Provider (chat, delegated
// to provider/openai) and imagegen.Provider (image generation), and exposes // to provider/openai) and imagegen.Provider (image generation), and exposes
// llama-swap's management endpoints as concrete methods. // llama-swap's management endpoints as concrete methods.
+100
View File
@@ -0,0 +1,100 @@
// sfx.go implements a second musicgen.Model factory against a Stable Audio
// Open shim (sfxgen) reached through llama-swap's /upstream passthrough
// (ADR-0024):
//
// POST /upstream/<id>/v1/sfx JSON {prompt, seconds?, steps?, cfg_scale?, seed?}
//
// Unlike the ACE-Step music path (music.go), /v1/sfx is SYNCHRONOUS: the
// response body is the finished WAV clip — no job queue, no polling. The
// surface reuses the musicgen types (a sound effect is a short audio clip
// from a text prompt); only the provider method differs.
package llamaswap
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
)
// SFXModel returns a musicgen.Model bound to a sync sound-effect backend.
// The id selects which upstream llama-swap loads.
func (p *Provider) SFXModel(id string, opts ...musicgen.ModelOption) (musicgen.Model, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = musicgen.ApplyModelOptions(opts)
return &sfxModel{p: p, id: id}, nil
}
type sfxModel struct {
p *Provider
id string
}
// sfxRequest is the sfxgen shim's /v1/sfx shape. Optional fields stay off
// the wire so the model's own defaults apply.
type sfxRequest struct {
Prompt string `json:"prompt"`
Seconds int `json:"seconds,omitempty"`
Steps *int `json:"steps,omitempty"`
CFGScale *float64 `json:"cfg_scale,omitempty"`
Seed *int64 `json:"seed,omitempty"`
}
// Generate implements musicgen.Model. The clip-length ceiling (~11s for
// Stable Audio Open Small) is the backend's to enforce, not this client's.
func (m *sfxModel) Generate(ctx context.Context, req musicgen.Request, opts ...musicgen.Option) (*musicgen.Result, error) {
req = req.Apply(opts...)
if strings.TrimSpace(req.Prompt) == "" {
return nil, fmt.Errorf("%w: sfx generation requires a prompt", llm.ErrUnsupported)
}
if req.Lyrics != "" {
return nil, fmt.Errorf("%w: sfx generation does not support lyrics", llm.ErrUnsupported)
}
if req.DurationSeconds < 0 {
return nil, fmt.Errorf("%w: duration must be >= 0, got %d", llm.ErrUnsupported, req.DurationSeconds)
}
if req.Steps != nil && *req.Steps <= 0 {
return nil, fmt.Errorf("%w: inference steps must be > 0, got %d", llm.ErrUnsupported, *req.Steps)
}
// The endpoint emits WAV only; a caller asking for another container
// would silently get mislabelled bytes — reject instead.
if f := strings.ToLower(strings.TrimSpace(req.Format)); f != "" && f != "wav" {
return nil, fmt.Errorf("%w: sfx output is wav only, got format %q", llm.ErrUnsupported, req.Format)
}
path, err := upstreamPath(m.id, "/v1/sfx")
if err != nil {
return nil, err
}
wire := sfxRequest{
Prompt: req.Prompt,
Seconds: req.DurationSeconds,
Steps: req.Steps,
CFGScale: req.CFGScale,
Seed: req.Seed,
}
encoded, err := json.Marshal(wire)
if err != nil {
return nil, fmt.Errorf("llama-swap: encode sfx request: %w", err)
}
raw, contentType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, "application/json", bytes.NewReader(encoded), maxAudioResponseBytes)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "sfx response contained no audio"}
}
mimeType := audioResultMIME(contentType, raw)
if mimeType == "" {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("sfx response is not audio (Content-Type %q): %s", contentType, truncateForError(raw))}
}
return &musicgen.Result{Audio: musicgen.Audio{Data: raw, MIME: mimeType}}, nil
}
+165
View File
@@ -0,0 +1,165 @@
// stems.go implements audio.StemSeparationProvider against a Demucs shim
// (audioutils) reached through llama-swap's /upstream passthrough (ADR-0024):
//
// POST /upstream/<id>/v1/stems multipart file[,model,two_stems,format]
//
// The response is a ZIP of the separated stems — one entry per stem, entry
// name = stem name, extension = container. Zip transport keeps a 4-stem WAV
// result (hundreds of MB decoded) off the JSON-of-base64 path entirely.
package llamaswap
import (
"archive/zip"
"bytes"
"context"
"fmt"
"io"
"net/http"
"path"
"strings"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)
// maxStemsResponseBytes caps the /v1/stems zip body: four WAV stems of a
// long song legitimately pass the 64MB JSON cap. Bounded so a buggy
// upstream can't allocate without limit.
const maxStemsResponseBytes = 512 << 20
Review

🟠 Excessive 512MB response cap enables DoS via concurrent large requests

security · flagged by 1 model

  • provider/llamaswap/stems.go:28512MB response cap is excessive for a production service. While bounded, 512MB per request allows a single caller to consume significant server memory. This is a DoS vector if multiple concurrent requests are permitted. Fix: Reduce to a more conservative limit (e.g., 64-128MB) unless there's a documented requirement for full-song stems at this size. Verified: Read stems.go:25-28 — constant is 512 << 20 (512MB). Comment states "four WAV stems o…

🪰 Gadfly · advisory

🟠 **Excessive 512MB response cap enables DoS via concurrent large requests** _security · flagged by 1 model_ - `provider/llamaswap/stems.go:28` — **512MB response cap is excessive for a production service.** While bounded, 512MB per request allows a single caller to consume significant server memory. This is a DoS vector if multiple concurrent requests are permitted. **Fix:** Reduce to a more conservative limit (e.g., 64-128MB) unless there's a documented requirement for full-song stems at this size. **Verified:** Read `stems.go:25-28` — constant is `512 << 20` (512MB). Comment states "four WAV stems o… <sub>🪰 Gadfly · advisory</sub>
// maxStemEntryBytes caps ONE decompressed zip entry — the zip-bomb guard.
// A single WAV stem of even a very long song stays far under this.
const maxStemEntryBytes = 256 << 20
// maxStemEntries caps how many stem entries are unpacked: Demucs emits at
// most six, so anything past a small multiple of that is a hostile or
// broken archive, not a result.
const maxStemEntries = 16
// maxStemsTotalBytes caps the AGGREGATE decompressed size across entries —
// the per-entry bound alone would still let a many-entry bomb multiply up.
const maxStemsTotalBytes = 1 << 30
// StemSeparatorModel implements audio.StemSeparationProvider. The id selects
// which upstream llama-swap loads.
func (p *Provider) StemSeparatorModel(id string, opts ...audio.StemSeparatorModelOption) (audio.StemSeparator, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = audio.ApplyStemSeparatorModelOptions(opts)
return &stemSeparatorModel{p: p, id: id}, nil
}
type stemSeparatorModel struct {
p *Provider
id string
}
// SeparateStems implements audio.StemSeparator.
func (m *stemSeparatorModel) SeparateStems(ctx context.Context, req audio.StemSeparationRequest, opts ...audio.StemSeparationOption) (*audio.StemSeparationResult, error) {
req = req.Apply(opts...)
if len(req.Audio) == 0 {
return nil, fmt.Errorf("%w: stem separation requires audio bytes", llm.ErrUnsupported)
}
mode := strings.ToLower(strings.TrimSpace(req.Mode))
if mode != "" && mode != "two" && mode != "four" {
return nil, fmt.Errorf("%w: stem mode must be \"two\" or \"four\", got %q", llm.ErrUnsupported, req.Mode)
}
format := strings.ToLower(strings.TrimSpace(req.Format))
if format != "" && format != "mp3" && format != "wav" {
return nil, fmt.Errorf("%w: stem format must be \"mp3\" or \"wav\", got %q", llm.ErrUnsupported, req.Format)
}
upPath, err := upstreamPath(m.id, "/v1/stems")
if err != nil {
return nil, err
Review

🟠 buildMultipart copies full audio into bytes.Buffer, doubling peak memory for large stem-separation inputs

performance · flagged by 1 model

🪰 Gadfly · advisory

🟠 **buildMultipart copies full audio into bytes.Buffer, doubling peak memory for large stem-separation inputs** _performance · flagged by 1 model_ <sub>🪰 Gadfly · advisory</sub>
}
// Demucs' two-stem mode is "isolate one source vs the rest"; vocals is
// the split this surface promises ("two" = vocals + accompaniment).
twoStems := ""
if mode == "two" {
twoStems = "vocals"
}
body, contentType, err := buildMultipart("build stems form",
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
[]formField{
{"model", req.Model, false},
{"two_stems", twoStems, false},
{"format", format, false},
})
if err != nil {
return nil, err
}
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, upPath, m.id, contentType, body, maxStemsResponseBytes)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems response contained no data"}
Review

🔴 Zip-bomb guard caps per-entry size but not total decompressed size or entry count, allowing unbounded memory growth in res.Stems

error-handling, performance, security · flagged by 3 models

  • provider/llamaswap/stems.go:97-116 — zip-bomb guard is incomplete: total decompressed size is unbounded. The unpack loop (lines 97-116) caps each individual entry at maxStemEntryBytes (256 MB via readZipEntry at lines 125-139) and caps the compressed zip body at maxStemsResponseBytes (512 MB, line 83), but it accumulates every entry's decompressed bytes into res.Stems with no running total and no entry-count limit. Deflate achieves high compression ratios on compressible in…

🪰 Gadfly · advisory

🔴 **Zip-bomb guard caps per-entry size but not total decompressed size or entry count, allowing unbounded memory growth in res.Stems** _error-handling, performance, security · flagged by 3 models_ - **`provider/llamaswap/stems.go:97-116` — zip-bomb guard is incomplete: total decompressed size is unbounded.** The unpack loop (lines 97-116) caps each *individual* entry at `maxStemEntryBytes` (256 MB via `readZipEntry` at lines 125-139) and caps the *compressed* zip body at `maxStemsResponseBytes` (512 MB, line 83), but it accumulates *every* entry's decompressed bytes into `res.Stems` with no running total and no entry-count limit. Deflate achieves high compression ratios on compressible in… <sub>🪰 Gadfly · advisory</sub>
}
zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
if err != nil {
// A non-zip 2xx body is a misconfigured upstream (an HTML error page
// behind a proxy, a JSON soft error) — fail loud, quoting a slice.
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("stems response is not a zip (Content-Type %q): %s", respType, truncateForError(raw))}
}
res := &audio.StemSeparationResult{}
var totalBytes int64
for _, f := range zr.File {
if f.FileInfo().IsDir() {
continue
}
if len(res.Stems) >= maxStemEntries {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("stems zip holds more than %d entries", maxStemEntries)}
}
data, err := readZipEntry(f)
if err != nil {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("stems zip entry %q: %v", f.Name, err)}
}
totalBytes += int64(len(data))
if totalBytes > maxStemsTotalBytes {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("stems zip decompresses past %d bytes", int64(maxStemsTotalBytes))}
}
// Entry name → stem name; extension → MIME. Entries may sit in a
// per-model directory ("htdemucs/vocals.mp3"), so use the base name.
base := path.Base(f.Name)
ext := path.Ext(base)
res.Stems = append(res.Stems, audio.Stem{
Name: strings.TrimSuffix(base, ext),
Audio: data,
MIME: stemMIME(ext),
})
}
if len(res.Stems) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems zip contained no stems"}
}
return res, nil
}
// readZipEntry decompresses one entry, bounded by maxStemEntryBytes so a
// zip bomb can't allocate without limit.
func readZipEntry(f *zip.File) ([]byte, error) {
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
data, err := io.ReadAll(io.LimitReader(rc, maxStemEntryBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > maxStemEntryBytes {
return nil, fmt.Errorf("decompressed entry exceeds %d bytes", int64(maxStemEntryBytes))
}
return data, nil
}
// stemMIME maps a stem file extension to its MIME type, reusing speechMIME's
// format table ("" and unknown extensions land on the mp3 default — Demucs'
// own default container).
func stemMIME(ext string) string {
return speechMIME("", strings.TrimPrefix(strings.ToLower(ext), "."))
}
@@ -0,0 +1,355 @@
package llamaswap
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
)
// stemsZipFixture builds a Demucs-style stems zip: entries under a per-model
// directory, entry name = stem name, extension = container.
func stemsZipFixture(t *testing.T, entries map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for name, data := range entries {
w, err := zw.Create(name)
if err != nil {
t.Fatalf("zip create: %v", err)
}
if _, err := w.Write([]byte(data)); err != nil {
t.Fatalf("zip write: %v", err)
}
}
if err := zw.Close(); err != nil {
t.Fatalf("zip close: %v", err)
}
return buf.Bytes()
}
func TestSeparateStems(t *testing.T) {
zipBody := stemsZipFixture(t, map[string]string{
"htdemucs/vocals.mp3": "VOX",
"htdemucs/no_vocals.mp3": "ACC",
})
var gotPath, gotTwoStems, gotModel, gotFormat, gotFilename string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotTwoStems = r.FormValue("two_stems")
gotModel = r.FormValue("model")
gotFormat = r.FormValue("format")
if _, hdr, err := r.FormFile("file"); err == nil {
gotFilename = hdr.Filename
} else {
t.Errorf("file part: %v", err)
}
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(zipBody)
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ss, err := p.StemSeparatorModel("audioutils")
if err != nil {
t.Fatalf("StemSeparatorModel: %v", err)
}
res, err := ss.SeparateStems(context.Background(),
audio.StemSeparationRequest{Audio: []byte("SONG"), MIME: "audio/mpeg"},
audio.WithStemMode("two"), audio.WithStemModel("htdemucs_ft"), audio.WithStemFormat("mp3"))
if err != nil {
t.Fatalf("SeparateStems: %v", err)
}
if gotPath != "/upstream/audioutils/v1/stems" {
t.Errorf("path = %q", gotPath)
}
if gotTwoStems != "vocals" || gotModel != "htdemucs_ft" || gotFormat != "mp3" || gotFilename != "audio.mp3" {
t.Errorf("two_stems/model/format/filename = %q/%q/%q/%q", gotTwoStems, gotModel, gotFormat, gotFilename)
}
if len(res.Stems) != 2 {
t.Fatalf("stems = %+v", res.Stems)
}
byName := map[string]audio.Stem{}
for _, s := range res.Stems {
byName[s.Name] = s
}
if v := byName["vocals"]; string(v.Audio) != "VOX" || v.MIME != "audio/mpeg" {
t.Errorf("vocals = %+v", v)
}
if a := byName["no_vocals"]; string(a.Audio) != "ACC" || a.MIME != "audio/mpeg" {
t.Errorf("no_vocals = %+v", a)
}
}
func TestSeparateStemsOmitsUnsetFields(t *testing.T) {
zipBody := stemsZipFixture(t, map[string]string{"vocals.wav": "V"})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
for _, k := range []string{"two_stems", "model", "format"} {
if v, ok := r.MultipartForm.Value[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
_, _ = w.Write(zipBody)
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ss, _ := p.StemSeparatorModel("audioutils")
res, err := ss.SeparateStems(context.Background(),
audio.StemSeparationRequest{Audio: []byte("SONG")})
if err != nil {
t.Fatalf("SeparateStems: %v", err)
}
// Top-level entry, wav extension.
if len(res.Stems) != 1 || res.Stems[0].Name != "vocals" || res.Stems[0].MIME != "audio/wav" {
t.Errorf("stems = %+v", res.Stems)
}
}
func TestSeparateStemsRejectsBadArgs(t *testing.T) {
p := New(WithBaseURL("http://unused"))
ss, _ := p.StemSeparatorModel("audioutils")
if _, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("no audio: err = %v, want ErrUnsupported", err)
}
if _, err := ss.SeparateStems(context.Background(),
audio.StemSeparationRequest{Audio: []byte{1}, Mode: "three"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("mode three: err = %v, want ErrUnsupported", err)
}
if _, err := ss.SeparateStems(context.Background(),
audio.StemSeparationRequest{Audio: []byte{1}, Format: "flac"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("format flac: err = %v, want ErrUnsupported", err)
}
}
func TestSeparateStemsRejectsNonZip(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("<html>proxy error page</html>"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ss, _ := p.StemSeparatorModel("audioutils")
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for non-zip body", err)
}
}
func TestSeparateStemsRejectsEmptyZip(t *testing.T) {
zipBody := stemsZipFixture(t, map[string]string{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(zipBody)
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ss, _ := p.StemSeparatorModel("audioutils")
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for stem-less zip", err)
}
}
func TestSeparateStemsRejectsTooManyEntries(t *testing.T) {
Review

wavFixture test helper defined in stems_sfx_enhance_test.go but used by clone_translate_test.go — poor test helper locality

maintainability · flagged by 1 model

  • wavFixture() defined in stems_sfx_enhance_test.go but consumed by clone_translate_test.go (provider/llamaswap/stems_sfx_enhance_test.go:174, provider/llamaswap/clone_translate_test.go:31): Same locality problem in tests — a shared test helper lives in a file whose name suggests it belongs to stems/sfx/enhance tests only. Since both files are in package llamaswap it compiles, but a maintainer adding a new test that needs a WAV fixture won't find it without a grep. Move it to a d…

🪰 Gadfly · advisory

⚪ **wavFixture test helper defined in stems_sfx_enhance_test.go but used by clone_translate_test.go — poor test helper locality** _maintainability · flagged by 1 model_ - **`wavFixture()` defined in `stems_sfx_enhance_test.go` but consumed by `clone_translate_test.go`** (`provider/llamaswap/stems_sfx_enhance_test.go:174`, `provider/llamaswap/clone_translate_test.go:31`): Same locality problem in tests — a shared test helper lives in a file whose name suggests it belongs to stems/sfx/enhance tests only. Since both files are in `package llamaswap` it compiles, but a maintainer adding a new test that needs a WAV fixture won't find it without a grep. Move it to a d… <sub>🪰 Gadfly · advisory</sub>
entries := map[string]string{}
for i := 0; i <= maxStemEntries; i++ {
entries[fmt.Sprintf("stem%02d.wav", i)] = "X"
}
zipBody := stemsZipFixture(t, entries)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(zipBody)
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ss, _ := p.StemSeparatorModel("audioutils")
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) || !strings.Contains(apiErr.Message, "entries") {
t.Fatalf("err = %v, want APIError for over-long stems zip", err)
}
}
// wavFixture is a minimal RIFF/WAVE header so http.DetectContentType sniffs
// audio/wave.
func wavFixture() []byte {
return []byte("RIFF\x24\x00\x00\x00WAVEfmt ")
}
func TestSFXGenerate(t *testing.T) {
var gotPath string
var gotBody map[string]any
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
_ = json.NewDecoder(r.Body).Decode(&gotBody)
w.Header().Set("Content-Type", "audio/wav")
_, _ = w.Write(wavFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, err := p.SFXModel("sfxgen-stableaudio")
if err != nil {
t.Fatalf("SFXModel: %v", err)
}
res, err := sm.Generate(context.Background(),
musicgen.Request{Prompt: "glass shattering", DurationSeconds: 8},
musicgen.WithSteps(50), musicgen.WithCFGScale(7), musicgen.WithSeed(42))
if err != nil {
t.Fatalf("Generate: %v", err)
}
if gotPath != "/upstream/sfxgen-stableaudio/v1/sfx" {
t.Errorf("path = %q", gotPath)
}
want := map[string]any{"prompt": "glass shattering", "seconds": 8.0, "steps": 50.0, "cfg_scale": 7.0, "seed": 42.0}
for k, w := range want {
if gotBody[k] != w {
t.Errorf("%s = %v, want %v", k, gotBody[k], w)
}
}
if res.Audio.MIME != "audio/wav" || len(res.Audio.Data) == 0 {
t.Errorf("audio = %q/%d bytes", res.Audio.MIME, len(res.Audio.Data))
}
}
func TestSFXOmitsDefaults(t *testing.T) {
var gotBody map[string]any
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewDecoder(r.Body).Decode(&gotBody)
// No Content-Type: the RIFF sniff must still label the clip.
w.Header()["Content-Type"] = nil
_, _ = w.Write(wavFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SFXModel("sfxgen-stableaudio")
res, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"})
if err != nil {
t.Fatalf("Generate: %v", err)
}
for _, k := range []string{"seconds", "steps", "cfg_scale", "seed"} {
if v, ok := gotBody[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
if res.Audio.MIME != "audio/wav" {
t.Errorf("MIME = %q, want sniffed audio/wav", res.Audio.MIME)
}
}
func TestSFXRejectsBadArgs(t *testing.T) {
p := New(WithBaseURL("http://unused"))
sm, _ := p.SFXModel("sfxgen-stableaudio")
if _, err := sm.Generate(context.Background(), musicgen.Request{Prompt: " "}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("empty prompt: err = %v, want ErrUnsupported", err)
}
if _, err := sm.Generate(context.Background(),
musicgen.Request{Prompt: "boom", Lyrics: "la la"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("lyrics: err = %v, want ErrUnsupported", err)
}
if _, err := sm.Generate(context.Background(),
musicgen.Request{Prompt: "boom", Format: "mp3"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("format mp3: err = %v, want ErrUnsupported", err)
}
}
func TestSFXRejectsNonAudioResponse(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"detail":"queue full"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SFXModel("sfxgen-stableaudio")
_, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for non-audio body", err)
}
}
func TestEnhance(t *testing.T) {
var gotPath, gotFilename string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
if _, hdr, err := r.FormFile("file"); err == nil {
gotFilename = hdr.Filename
} else {
t.Errorf("file part: %v", err)
}
w.Header().Set("Content-Type", "audio/wav")
_, _ = w.Write(wavFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
en, err := p.SpeechEnhancerModel("audioutils")
if err != nil {
t.Fatalf("SpeechEnhancerModel: %v", err)
}
res, err := en.Enhance(context.Background(),
audio.EnhancementRequest{Audio: []byte("NOISY"), MIME: "audio/ogg"})
if err != nil {
t.Fatalf("Enhance: %v", err)
}
if gotPath != "/upstream/audioutils/v1/enhance" {
t.Errorf("path = %q", gotPath)
}
if gotFilename != "audio.ogg" {
t.Errorf("filename = %q", gotFilename)
}
if res.MIME != "audio/wav" || len(res.Audio) == 0 {
t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio))
}
}
func TestEnhanceRejectsEmptyAudio(t *testing.T) {
p := New(WithBaseURL("http://unused"))
en, _ := p.SpeechEnhancerModel("audioutils")
if _, err := en.Enhance(context.Background(), audio.EnhancementRequest{}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("err = %v, want ErrUnsupported", err)
}
}
func TestEnhanceRejectsNonAudioResponse(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("<html>oops</html>"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
en, _ := p.SpeechEnhancerModel("audioutils")
_, err := en.Enhance(context.Background(), audio.EnhancementRequest{Audio: []byte("NOISY")})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for non-audio body", err)
}
}