From b3a172a0532479862dbb074494df4fcd910829ca Mon Sep 17 00:00:00 2001 From: Steve Dudenhoeffer Date: Thu, 16 Jul 2026 17:01:11 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20wave-3=20audio=20surfaces=20=E2=80=94?= =?UTF-8?q?=20stems,=20sfx,=20speech=20enhance,=20voice=20clone,=20transla?= =?UTF-8?q?te=20(ADR-0024)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - audio.StemSeparator/StemSeparationProvider: Demucs zip transport via POST /upstream//v1/stems (Mode two -> two_stems=vocals; model + format fields); bounded zip unpack, entry name -> stem, ext -> MIME. - SFXModel reuses musicgen against the sync /upstream//v1/sfx route (JSON prompt/seconds/steps/cfg_scale/seed -> WAV); musicgen.Request gains CFGScale. - audio.SpeechEnhancer/SpeechEnhancementProvider: POST /upstream//v1/enhance -> WAV (result reuses SpeechResult). - SpeechRequest.ReferenceAudio/ReferenceMIME (+WithReferenceAudio): llamaswap switches to the chatterbox clone route POST /upstream//v1/audio/speech/upload (input + voice_file), wav MIME fallback. - TranscriptionRequest.Translate (+WithTranslate): translate=true form field, language=auto forced when no explicit hint (whisper.cpp default en would skip translation). - httptest contract tests (zip unpack, clone-route switch, translate + auto-language injection); ADR-0024 (index row deferred — MJ-A backfills the ADR index table and parallel edits would conflict). Co-Authored-By: Claude Fable 5 --- audio/audio.go | 28 ++ audio/enhance.go | 66 ++++ audio/stems.go | 113 +++++++ docs/adr/0024-audio-wave3-surfaces.md | 65 ++++ musicgen/musicgen.go | 8 + provider/llamaswap/audio.go | 68 +++- provider/llamaswap/clone_translate_test.go | 179 ++++++++++ provider/llamaswap/enhance.go | 61 ++++ provider/llamaswap/sfx.go | 118 +++++++ provider/llamaswap/stems.go | 146 ++++++++ provider/llamaswap/stems_sfx_enhance_test.go | 332 +++++++++++++++++++ 11 files changed, 1182 insertions(+), 2 deletions(-) create mode 100644 audio/enhance.go create mode 100644 audio/stems.go create mode 100644 docs/adr/0024-audio-wave3-surfaces.md create mode 100644 provider/llamaswap/clone_translate_test.go create mode 100644 provider/llamaswap/enhance.go create mode 100644 provider/llamaswap/sfx.go create mode 100644 provider/llamaswap/stems.go create mode 100644 provider/llamaswap/stems_sfx_enhance_test.go diff --git a/audio/audio.go b/audio/audio.go index ec60db4..c97505d 100644 --- a/audio/audio.go +++ b/audio/audio.go @@ -30,6 +30,17 @@ type SpeechRequest struct { // Speed is the playback-rate multiplier; 0 = backend default (1.0). Speed float64 + + // ReferenceAudio is a short voice sample for zero-shot voice cloning + // (chatterbox style): when set, the model speaks Input in the sampled + // voice instead of a named Voice. nil = normal synthesis. Backends + // without cloning support must reject a reference-carrying request + // rather than silently ignoring it. + ReferenceAudio []byte + + // ReferenceMIME is the reference audio's MIME type (e.g. "audio/wav"); + // "" = let the backend sniff it. + ReferenceMIME string } // SpeechResult is the canonical synthesis result: raw audio bytes plus the @@ -59,6 +70,11 @@ func WithFormat(f string) SpeechOption { return func(r *SpeechRequest) { r.Forma // WithSpeed sets the playback-rate multiplier. func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } } +// WithReferenceAudio provides a voice sample for zero-shot voice cloning. +func WithReferenceAudio(data []byte, mime string) SpeechOption { + return func(r *SpeechRequest) { r.ReferenceAudio, r.ReferenceMIME = data, mime } +} + // Apply returns a copy of the request with all options applied. Providers // call this once at the top of Speak. func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest { @@ -121,6 +137,13 @@ type TranscriptionRequest struct { // Prompt is optional context or vocabulary to bias decoding; "" = none. Prompt string + + // Translate requests an English translation of the speech instead of a + // same-language transcript (whisper's translate task). When set and no + // Language is given, providers must force source-language auto-detection + // — a backend whose default language is "en" would otherwise skip + // translation entirely. + Translate bool } // TranscriptionResult is the canonical transcription result. @@ -145,6 +168,11 @@ func WithPrompt(p string) TranscriptionOption { return func(r *TranscriptionRequest) { r.Prompt = p } } +// WithTranslate requests an English translation instead of a transcript. +func WithTranslate() TranscriptionOption { + return func(r *TranscriptionRequest) { r.Translate = true } +} + // Apply returns a copy of the request with all options applied. func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest { for _, opt := range opts { diff --git a/audio/enhance.go b/audio/enhance.go new file mode 100644 index 0000000..ba14055 --- /dev/null +++ b/audio/enhance.go @@ -0,0 +1,66 @@ +package audio + +import "context" + +// EnhancementRequest asks a speech-enhancement backend (DeepFilterNet style) +// to denoise a recording. Audio is carried as bytes (never a URL), mirroring +// TranscriptionRequest (ADR-0024). +type EnhancementRequest struct { + // Audio is the encoded audio to enhance. + Audio []byte + + // MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend + // sniff it. + MIME string + + // Filename is the multipart filename hint some backends key their format + // detection on; "" derives one from MIME or falls back to "audio". + Filename string +} + +// EnhancementOption mutates an EnhancementRequest before it is sent. +// Reserved for future request settings (the reference backend takes no +// parameters). +type EnhancementOption func(*EnhancementRequest) + +// Apply returns a copy of the request with all options applied. +func (r EnhancementRequest) Apply(opts ...EnhancementOption) EnhancementRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// SpeechEnhancer denoises speech recordings. The result reuses SpeechResult +// (audio bytes + MIME + Raw) — enhancement is audio-in/audio-out exactly +// like synthesis. +type SpeechEnhancer interface { + // Enhance returns the denoised audio. + Enhance(ctx context.Context, req EnhancementRequest, opts ...EnhancementOption) (*SpeechResult, error) +} + +// SpeechEnhancerModelOption configures a SpeechEnhancer at construction time. +// Reserved for future per-model settings. +type SpeechEnhancerModelOption func(*SpeechEnhancerModelConfig) + +// SpeechEnhancerModelConfig carries per-model construction settings. +type SpeechEnhancerModelConfig struct{} + +// ApplySpeechEnhancerModelOptions folds options into a config. +func ApplySpeechEnhancerModelOptions(opts []SpeechEnhancerModelOption) SpeechEnhancerModelConfig { + var cfg SpeechEnhancerModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// SpeechEnhancementProvider mints SpeechEnhancers bound to one backend. +type SpeechEnhancementProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // SpeechEnhancerModel returns a SpeechEnhancer bound to the given id + // (passed through to the backend verbatim; no catalog validation). + SpeechEnhancerModel(id string, opts ...SpeechEnhancerModelOption) (SpeechEnhancer, error) +} diff --git a/audio/stems.go b/audio/stems.go new file mode 100644 index 0000000..291eb11 --- /dev/null +++ b/audio/stems.go @@ -0,0 +1,113 @@ +package audio + +import "context" + +// StemSeparationRequest asks a source-separation backend (Demucs style) to +// split a mix into stems. Audio is carried as bytes (never a URL), mirroring +// TranscriptionRequest. Zero values mean "backend default" (ADR-0024). +type StemSeparationRequest struct { + // Audio is the encoded mix to separate. + Audio []byte + + // MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend + // sniff it. + MIME string + + // Filename is the multipart filename hint some backends key their format + // detection on; "" derives one from MIME or falls back to "audio". + Filename string + + // Mode selects the split: "two" (vocals + accompaniment) or "four" + // (vocals/drums/bass/other); "" = backend default (four). + Mode string + + // Model selects the separator's internal network where the backend + // offers several (Demucs: "htdemucs", "htdemucs_ft"); "" = backend + // default. This is NOT the provider model id — that is fixed when the + // StemSeparator is minted (mirrors BackgroundRemovalRequest.Net). + Model string + + // Format is the per-stem audio container ("mp3" or "wav"); + // "" = backend default. + Format string +} + +// Stem is one separated source. +type Stem struct { + // Name is the stem's name ("vocals", "drums", "bass", "other", + // "no_vocals", ...), taken from the backend's own labelling. + Name string + + // Audio is the encoded stem. + Audio []byte + + // MIME is the stem's audio MIME type, e.g. "audio/mpeg". + MIME string +} + +// StemSeparationResult is the canonical separation result. +type StemSeparationResult struct { + // Stems are the separated sources, in the order the backend returned + // them. + Stems []Stem +} + +// StemSeparationOption mutates a StemSeparationRequest before it is sent. +type StemSeparationOption func(*StemSeparationRequest) + +// WithStemMode selects the split ("two" or "four"). +func WithStemMode(m string) StemSeparationOption { + return func(r *StemSeparationRequest) { r.Mode = m } +} + +// WithStemModel selects the separator's internal network. +func WithStemModel(m string) StemSeparationOption { + return func(r *StemSeparationRequest) { r.Model = m } +} + +// WithStemFormat sets the per-stem audio container ("mp3", "wav"). +func WithStemFormat(f string) StemSeparationOption { + return func(r *StemSeparationRequest) { r.Format = f } +} + +// Apply returns a copy of the request with all options applied. +func (r StemSeparationRequest) Apply(opts ...StemSeparationOption) StemSeparationRequest { + for _, opt := range opts { + opt(&r) + } + return r +} + +// StemSeparator splits a mix into stems. +type StemSeparator interface { + // SeparateStems returns the separated sources. Separation is CPU-bound + // and slow (minutes for a full song); bound the call with a context + // deadline. + SeparateStems(ctx context.Context, req StemSeparationRequest, opts ...StemSeparationOption) (*StemSeparationResult, error) +} + +// StemSeparatorModelOption configures a StemSeparator at construction time. +// Reserved for future per-model settings. +type StemSeparatorModelOption func(*StemSeparatorModelConfig) + +// StemSeparatorModelConfig carries per-model construction settings. +type StemSeparatorModelConfig struct{} + +// ApplyStemSeparatorModelOptions folds options into a config. +func ApplyStemSeparatorModelOptions(opts []StemSeparatorModelOption) StemSeparatorModelConfig { + var cfg StemSeparatorModelConfig + for _, opt := range opts { + opt(&cfg) + } + return cfg +} + +// StemSeparationProvider mints StemSeparators bound to one backend. +type StemSeparationProvider interface { + // Name is the registry identifier for the provider. + Name() string + + // StemSeparatorModel returns a StemSeparator bound to the given id + // (passed through to the backend verbatim; no catalog validation). + StemSeparatorModel(id string, opts ...StemSeparatorModelOption) (StemSeparator, error) +} diff --git a/docs/adr/0024-audio-wave3-surfaces.md b/docs/adr/0024-audio-wave3-surfaces.md new file mode 100644 index 0000000..a96361b --- /dev/null +++ b/docs/adr/0024-audio-wave3-surfaces.md @@ -0,0 +1,65 @@ +# ADR-0024: Wave-3 audio surfaces (stems, SFX, speech enhance, voice clone, translate) + +Status: Accepted (2026-07-16) + +## Context + +The llama-swap host is gaining wave-3 audio capabilities (spec: mort +docs/specs/2026-07-16-llamaswap-wave3.md): Demucs stem separation and +DeepFilterNet speech enhancement (both on the CPU `audioutils` shim), Stable +Audio Open sound effects (`sfxgen`), plus two upgrades to existing models — +chatterbox's stateless voice-clone route and whisper.cpp's per-request +translate flag (both verified against the live images 2026-07-16). + +## Decision + +1. **`audio.StemSeparator` / `StemSeparationProvider`** — + `StemSeparationRequest{Audio, MIME, Filename, Mode, Model, Format}` → + `StemSeparationResult{Stems []Stem{Name, Audio, MIME}}`. + - `Mode` is the caller-facing split: `"two"` (vocals + accompaniment, + sent as Demucs' `two_stems=vocals`) or `"four"`; `""` = backend + default. `Model` selects the Demucs variant (`htdemucs`/`htdemucs_ft`), + mirroring `BackgroundRemovalRequest.Net`. + - **The wire format is a ZIP** (`POST /upstream//v1/stems`): four WAV + stems would blow any JSON-of-base64 budget. Entry name → stem name, + extension → MIME; entries may sit under a per-model directory. Unpacking + is bounded per entry (zip-bomb guard) and a non-zip 2xx body fails loud. +2. **`SFXModel` reuses `musicgen`** — a sound effect is a short audio clip + from a text prompt; only the provider method differs. The sfxgen route + (`POST /upstream//v1/sfx`, JSON `{prompt, seconds, steps?, cfg_scale?, + seed?}`) is SYNCHRONOUS (WAV body), unlike ACE-Step's job queue. + `musicgen.Request` gains `CFGScale *float64` for it; lyrics and non-wav + formats are rejected (the model can't honor them). The ~11s model ceiling + is the backend's to enforce. +3. **`audio.SpeechEnhancer` / `SpeechEnhancementProvider`** — + `EnhancementRequest{Audio, MIME, Filename}` → + `POST /upstream//v1/enhance` → WAV. The result reuses `SpeechResult`; + audio-in/audio-out needs no new result type. +4. **Voice cloning is a `SpeechRequest` field, not a new surface** — + `ReferenceAudio []byte` + `ReferenceMIME`. When set, the llamaswap + speech model switches from JSON `/v1/audio/speech` to multipart + `POST /upstream//v1/audio/speech/upload` (fields `input` + + `voice_file`; verified live: stateless per-request cloning, no voice + library). Voice/format/speed still ride when set; the clone route's MIME + fallback is wav (not the JSON route's mp3). Backends without cloning must + reject a reference-carrying request rather than silently ignore it. +5. **Translation is a `TranscriptionRequest` bool** — `Translate` maps to + whisper.cpp's `translate=true` form field (server.cpp:534). Because that + server's language DEFAULT is `en` (which silently skips translation), the + provider forces `language=auto` when translating without an explicit + language hint; an explicit hint wins. +6. **Binary success bodies are validated before wrapping** (ADR-0020 rule): + sfx/enhance require positive evidence of audio-ness (declared audio/* + Content-Type or sniffed RIFF/WAVE), stems require a parseable zip with + at least one stem. + +## Consequences + +- `audio` grows from three surfaces to five; the SFX surface adds zero new + types (musicgen reuse) — one format→MIME table and one multipart builder + keep serving every audio endpoint. +- The clone-route switch means one `SpeechModel` can answer over two wire + shapes; tests pin both routes so a regression can't silently drop cloning. +- `two_stems` is hardwired to vocals: "isolate X vs the rest" for other + sources is a backend capability not exposed in v1 (add a field when a + caller needs it, not before). diff --git a/musicgen/musicgen.go b/musicgen/musicgen.go index e009f3c..f5dfd3e 100644 --- a/musicgen/musicgen.go +++ b/musicgen/musicgen.go @@ -40,6 +40,11 @@ type Request struct { // Steps is the number of inference steps; nil = backend default. Steps *int + // CFGScale is the classifier-free-guidance scale; nil = backend default. + // Architecture-sensitive, so prefer leaving it nil unless the caller + // knows the target model. Backends without the knob ignore it. + CFGScale *float64 + // Seed fixes the RNG seed for reproducible output; nil = backend // default (random). Seed *int64 @@ -73,6 +78,9 @@ func WithFormat(f string) Option { return func(r *Request) { r.Format = f } } // WithSteps overrides the number of inference steps. func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } } +// WithCFGScale overrides the classifier-free-guidance scale. +func WithCFGScale(s float64) Option { return func(r *Request) { r.CFGScale = &s } } + // WithSeed fixes the RNG seed. func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } } diff --git a/provider/llamaswap/audio.go b/provider/llamaswap/audio.go index 024ad17..3372250 100644 --- a/provider/llamaswap/audio.go +++ b/provider/llamaswap/audio.go @@ -9,6 +9,7 @@ import ( "mime" "net/http" "net/url" + "strconv" "strings" "gitea.stevedudenhoeffer.com/steve/majordomo/audio" @@ -42,7 +43,11 @@ type speechRequest struct { Speed float64 `json:"speed,omitempty"` } -// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech. +// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech, or — +// when the request carries reference audio for voice cloning — via multipart +// POST {base}/upstream//v1/audio/speech/upload (the chatterbox clone +// route; verified live 2026-07-16: stateless per-request cloning, fields +// `input` + `voice_file`). func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) { req = req.Apply(opts...) if strings.TrimSpace(req.Input) == "" { @@ -51,6 +56,9 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts . if req.Speed < 0 { return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed) } + if len(req.ReferenceAudio) > 0 { + return m.speakWithReference(ctx, req) + } wire := speechRequest{ Model: m.id, Input: req.Input, @@ -72,6 +80,49 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts . return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil } +// speakWithReference performs zero-shot voice cloning via the upstream +// passthrough clone route. The reference sample rides as the `voice_file` +// part and the text as an `input` field; voice/format/speed still go on the +// wire when set (upstreams ignore fields they don't understand). The route's +// default container is WAV, so MIME resolution falls back to audio/wav (not +// the JSON route's mp3) when neither the response header nor the requested +// format decides. +func (m *speechModel) speakWithReference(ctx context.Context, req audio.SpeechRequest) (*audio.SpeechResult, error) { + path, err := upstreamPath(m.id, "/v1/audio/speech/upload") + if err != nil { + return nil, err + } + speed := "" + if req.Speed != 0 { + speed = strconv.FormatFloat(req.Speed, 'g', -1, 64) + } + body, formType, err := buildMultipart("build speech clone form", + filePart{field: "voice_file", filename: transcriptionFilename("", req.ReferenceMIME), data: req.ReferenceAudio}, + []formField{ + {"input", req.Input, true}, + {"voice", req.Voice, false}, + {"response_format", req.Format, false}, + {"speed", speed, false}, + }) + if err != nil { + return nil, err + } + audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, formType, body, maxResponseBytes) + if err != nil { + return nil, err + } + if len(audioBytes) == 0 { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech clone response contained no audio"} + } + mimeType := "audio/wav" + if mt := mimeFromContentType(contentType, "audio/"); mt != "" { + mimeType = mt + } else if req.Format != "" { + mimeType = speechMIME("", req.Format) + } + return &audio.SpeechResult{Audio: audioBytes, MIME: mimeType}, nil +} + // speechMIME resolves the result MIME type: the response Content-Type when it // is a concrete audio type, else a mapping from the requested format, else // audio/mpeg (the OpenAI endpoint's default container is mp3). @@ -118,13 +169,26 @@ func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.Transcrip return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported) } + // Translation is a per-request bool form field on whisper.cpp's server + // (server.cpp:534, verified 2026-07-16). Its language DEFAULT is "en", + // which would make translate a no-op — so an explicit language hint + // wins, but an unset one is forced to auto-detection. + translate := "" + language := req.Language + if req.Translate { + translate = "true" + if language == "" { + language = "auto" + } + } buf, formType, err := buildMultipart("build transcription form", filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio}, []formField{ {"model", m.id, true}, {"response_format", "json", true}, - {"language", req.Language, false}, + {"language", language, false}, {"prompt", req.Prompt, false}, + {"translate", translate, false}, }) if err != nil { return nil, err diff --git a/provider/llamaswap/clone_translate_test.go b/provider/llamaswap/clone_translate_test.go new file mode 100644 index 0000000..788e425 --- /dev/null +++ b/provider/llamaswap/clone_translate_test.go @@ -0,0 +1,179 @@ +package llamaswap + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "testing" + + "gitea.stevedudenhoeffer.com/steve/majordomo/audio" +) + +func TestSpeakWithReferenceUsesCloneRoute(t *testing.T) { + var gotPath, gotInput, gotVoice, gotFilename string + var gotRef []byte + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + gotInput = r.FormValue("input") + gotVoice = r.FormValue("voice") + f, hdr, err := r.FormFile("voice_file") + if err != nil { + t.Fatalf("voice_file part: %v", err) + } + defer f.Close() + gotRef, _ = io.ReadAll(f) + gotFilename = hdr.Filename + w.Header().Set("Content-Type", "audio/wav") + _, _ = w.Write(wavFixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + sm, _ := p.SpeechModel("chatterbox") + res, err := sm.Speak(context.Background(), + audio.SpeechRequest{Input: "hello in my voice", Voice: "narrator"}, + audio.WithReferenceAudio([]byte("REFWAV"), "audio/wav")) + if err != nil { + t.Fatalf("Speak: %v", err) + } + if gotPath != "/upstream/chatterbox/v1/audio/speech/upload" { + t.Errorf("path = %q, want clone route", gotPath) + } + if gotInput != "hello in my voice" || gotVoice != "narrator" { + t.Errorf("input/voice = %q/%q", gotInput, gotVoice) + } + if string(gotRef) != "REFWAV" || gotFilename != "audio.wav" { + t.Errorf("ref = %q name = %q", gotRef, gotFilename) + } + if res.MIME != "audio/wav" || len(res.Audio) == 0 { + t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio)) + } +} + +func TestSpeakWithReferenceDefaultsToWavMIME(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + for _, k := range []string{"voice", "response_format", "speed"} { + if v, ok := r.MultipartForm.Value[k]; ok { + t.Errorf("unset request sent %q = %v, want omitted", k, v) + } + } + w.Header()["Content-Type"] = nil // no declared type at all + _, _ = w.Write([]byte("RAWAUDIO")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + sm, _ := p.SpeechModel("chatterbox") + res, err := sm.Speak(context.Background(), + audio.SpeechRequest{Input: "hi"}, + audio.WithReferenceAudio([]byte("REF"), "")) + if err != nil { + t.Fatalf("Speak: %v", err) + } + // The clone route's default container is wav, not the JSON route's mp3. + if res.MIME != "audio/wav" { + t.Errorf("MIME = %q, want audio/wav fallback", res.MIME) + } +} + +func TestSpeakWithoutReferenceKeepsJSONRoute(t *testing.T) { + var gotPath, gotContentType string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotContentType = r.Header.Get("Content-Type") + _, _ = w.Write([]byte("MP3")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + sm, _ := p.SpeechModel("chatterbox") + if _, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: "hi"}); err != nil { + t.Fatalf("Speak: %v", err) + } + if gotPath != "/v1/audio/speech" || gotContentType != "application/json" { + t.Errorf("path/content-type = %q/%q, want JSON route", gotPath, gotContentType) + } +} + +func TestTranscribeTranslateForcesAutoLanguage(t *testing.T) { + var gotTranslate, gotLanguage string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + gotTranslate = r.FormValue("translate") + gotLanguage = r.FormValue("language") + _, _ = w.Write([]byte(`{"text":"hello"}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + tm, _ := p.TranscriptionModel("whisper-large-v3-turbo") + res, err := tm.Transcribe(context.Background(), + audio.TranscriptionRequest{Audio: []byte("AUDIO")}, + audio.WithTranslate()) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + // whisper.cpp's server default language is "en", which would skip + // translation — translate must force auto-detection when no explicit + // language hint was given. + if gotTranslate != "true" || gotLanguage != "auto" { + t.Errorf("translate/language = %q/%q, want true/auto", gotTranslate, gotLanguage) + } + if res.Text != "hello" { + t.Errorf("text = %q", res.Text) + } +} + +func TestTranscribeTranslateKeepsExplicitLanguage(t *testing.T) { + var gotTranslate, gotLanguage string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + gotTranslate = r.FormValue("translate") + gotLanguage = r.FormValue("language") + _, _ = w.Write([]byte(`{"text":"hello"}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + tm, _ := p.TranscriptionModel("whisper-large-v3-turbo") + if _, err := tm.Transcribe(context.Background(), + audio.TranscriptionRequest{Audio: []byte("AUDIO"), Language: "de", Translate: true}); err != nil { + t.Fatalf("Transcribe: %v", err) + } + if gotTranslate != "true" || gotLanguage != "de" { + t.Errorf("translate/language = %q/%q, want true/de", gotTranslate, gotLanguage) + } +} + +func TestTranscribeWithoutTranslateOmitsFields(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + for _, k := range []string{"translate", "language"} { + if v, ok := r.MultipartForm.Value[k]; ok { + t.Errorf("unset request sent %q = %v, want omitted", k, v) + } + } + _, _ = w.Write([]byte(`{"text":"hi"}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + tm, _ := p.TranscriptionModel("whisper-large-v3-turbo") + if _, err := tm.Transcribe(context.Background(), + audio.TranscriptionRequest{Audio: []byte("AUDIO")}); err != nil { + t.Fatalf("Transcribe: %v", err) + } +} diff --git a/provider/llamaswap/enhance.go b/provider/llamaswap/enhance.go new file mode 100644 index 0000000..9509430 --- /dev/null +++ b/provider/llamaswap/enhance.go @@ -0,0 +1,61 @@ +// enhance.go implements audio.SpeechEnhancementProvider against a +// DeepFilterNet shim (audioutils) reached through llama-swap's /upstream +// passthrough (ADR-0024): +// +// POST /upstream//v1/enhance multipart file -> WAV +package llamaswap + +import ( + "context" + "fmt" + "net/http" + + "gitea.stevedudenhoeffer.com/steve/majordomo/audio" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" +) + +// SpeechEnhancerModel implements audio.SpeechEnhancementProvider. The id +// selects which upstream llama-swap loads. +func (p *Provider) SpeechEnhancerModel(id string, opts ...audio.SpeechEnhancerModelOption) (audio.SpeechEnhancer, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = audio.ApplySpeechEnhancerModelOptions(opts) + return &speechEnhancerModel{p: p, id: id}, nil +} + +type speechEnhancerModel struct { + p *Provider + id string +} + +// Enhance implements audio.SpeechEnhancer. The endpoint always answers WAV. +func (m *speechEnhancerModel) Enhance(ctx context.Context, req audio.EnhancementRequest, opts ...audio.EnhancementOption) (*audio.SpeechResult, error) { + req = req.Apply(opts...) + if len(req.Audio) == 0 { + return nil, fmt.Errorf("%w: speech enhancement requires audio bytes", llm.ErrUnsupported) + } + path, err := upstreamPath(m.id, "/v1/enhance") + if err != nil { + return nil, err + } + body, contentType, err := buildMultipart("build enhance form", + filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio}, + nil) + if err != nil { + return nil, err + } + raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "enhance response contained no audio"} + } + mimeType := audioResultMIME(respType, raw) + if mimeType == "" { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, + Message: fmt.Sprintf("enhance response is not audio (Content-Type %q): %s", respType, truncateForError(raw))} + } + return &audio.SpeechResult{Audio: raw, MIME: mimeType}, nil +} diff --git a/provider/llamaswap/sfx.go b/provider/llamaswap/sfx.go new file mode 100644 index 0000000..f0959e8 --- /dev/null +++ b/provider/llamaswap/sfx.go @@ -0,0 +1,118 @@ +// sfx.go implements a second musicgen.Model factory against a Stable Audio +// Open shim (sfxgen) reached through llama-swap's /upstream passthrough +// (ADR-0024): +// +// POST /upstream//v1/sfx JSON {prompt, seconds?, steps?, cfg_scale?, seed?} +// +// Unlike the ACE-Step music path (music.go), /v1/sfx is SYNCHRONOUS: the +// response body is the finished WAV clip — no job queue, no polling. The +// surface reuses the musicgen types (a sound effect is a short audio clip +// from a text prompt); only the provider method differs. +package llamaswap + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/musicgen" +) + +// SFXModel returns a musicgen.Model bound to a sync sound-effect backend. +// The id selects which upstream llama-swap loads. +func (p *Provider) SFXModel(id string, opts ...musicgen.ModelOption) (musicgen.Model, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = musicgen.ApplyModelOptions(opts) + return &sfxModel{p: p, id: id}, nil +} + +type sfxModel struct { + p *Provider + id string +} + +// sfxRequest is the sfxgen shim's /v1/sfx shape. Optional fields stay off +// the wire so the model's own defaults apply. +type sfxRequest struct { + Prompt string `json:"prompt"` + Seconds int `json:"seconds,omitempty"` + Steps *int `json:"steps,omitempty"` + CFGScale *float64 `json:"cfg_scale,omitempty"` + Seed *int64 `json:"seed,omitempty"` +} + +// Generate implements musicgen.Model. The clip-length ceiling (~11s for +// Stable Audio Open Small) is the backend's to enforce, not this client's. +func (m *sfxModel) Generate(ctx context.Context, req musicgen.Request, opts ...musicgen.Option) (*musicgen.Result, error) { + req = req.Apply(opts...) + if strings.TrimSpace(req.Prompt) == "" { + return nil, fmt.Errorf("%w: sfx generation requires a prompt", llm.ErrUnsupported) + } + if req.Lyrics != "" { + return nil, fmt.Errorf("%w: sfx generation does not support lyrics", llm.ErrUnsupported) + } + if req.DurationSeconds < 0 { + return nil, fmt.Errorf("%w: duration must be >= 0, got %d", llm.ErrUnsupported, req.DurationSeconds) + } + if req.Steps != nil && *req.Steps <= 0 { + return nil, fmt.Errorf("%w: inference steps must be > 0, got %d", llm.ErrUnsupported, *req.Steps) + } + // The endpoint emits WAV only; a caller asking for another container + // would silently get mislabelled bytes — reject instead. + if f := strings.ToLower(strings.TrimSpace(req.Format)); f != "" && f != "wav" { + return nil, fmt.Errorf("%w: sfx output is wav only, got format %q", llm.ErrUnsupported, req.Format) + } + + path, err := upstreamPath(m.id, "/v1/sfx") + if err != nil { + return nil, err + } + wire := sfxRequest{ + Prompt: req.Prompt, + Seconds: req.DurationSeconds, + Steps: req.Steps, + CFGScale: req.CFGScale, + Seed: req.Seed, + } + encoded, err := json.Marshal(wire) + if err != nil { + return nil, fmt.Errorf("llama-swap: encode sfx request: %w", err) + } + raw, contentType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, "application/json", bytes.NewReader(encoded), maxResponseBytes) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "sfx response contained no audio"} + } + mimeType := audioResultMIME(contentType, raw) + if mimeType == "" { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, + Message: fmt.Sprintf("sfx response is not audio (Content-Type %q): %s", contentType, truncateForError(raw))} + } + return &musicgen.Result{Audio: musicgen.Audio{Data: raw, MIME: mimeType}}, nil +} + +// audioResultMIME resolves an audio body's MIME type: the response +// Content-Type when it is a concrete audio type, else content sniffing +// (RIFF/WAVE and friends), else "" — the caller treats undetectable as an +// upstream error, mirroring videoMIME. Sniffed "audio/wave" is normalized +// to the conventional "audio/wav". +func audioResultMIME(contentType string, data []byte) string { + if mt := mimeFromContentType(contentType, "audio/"); mt != "" { + return mt + } + if mt := http.DetectContentType(data); strings.HasPrefix(mt, "audio/") { + if mt == "audio/wave" { + return "audio/wav" + } + return mt + } + return "" +} diff --git a/provider/llamaswap/stems.go b/provider/llamaswap/stems.go new file mode 100644 index 0000000..3a7fbb3 --- /dev/null +++ b/provider/llamaswap/stems.go @@ -0,0 +1,146 @@ +// stems.go implements audio.StemSeparationProvider against a Demucs shim +// (audioutils) reached through llama-swap's /upstream passthrough (ADR-0024): +// +// POST /upstream//v1/stems multipart file[,model,two_stems,format] +// +// The response is a ZIP of the separated stems — one entry per stem, entry +// name = stem name, extension = container. Zip transport keeps a 4-stem WAV +// result (hundreds of MB decoded) off the JSON-of-base64 path entirely. +package llamaswap + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "net/http" + "path" + "strings" + + "gitea.stevedudenhoeffer.com/steve/majordomo/audio" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" +) + +// maxStemsResponseBytes caps the /v1/stems zip body: four WAV stems of a +// long song legitimately pass the 64MB JSON cap. Bounded so a buggy +// upstream can't allocate without limit. +const maxStemsResponseBytes = 512 << 20 + +// maxStemEntryBytes caps ONE decompressed zip entry — the zip-bomb guard. +// A single WAV stem of even a very long song stays far under this. +const maxStemEntryBytes = 256 << 20 + +// StemSeparatorModel implements audio.StemSeparationProvider. The id selects +// which upstream llama-swap loads. +func (p *Provider) StemSeparatorModel(id string, opts ...audio.StemSeparatorModelOption) (audio.StemSeparator, error) { + if err := p.requireBaseURL(); err != nil { + return nil, err + } + _ = audio.ApplyStemSeparatorModelOptions(opts) + return &stemSeparatorModel{p: p, id: id}, nil +} + +type stemSeparatorModel struct { + p *Provider + id string +} + +// SeparateStems implements audio.StemSeparator. +func (m *stemSeparatorModel) SeparateStems(ctx context.Context, req audio.StemSeparationRequest, opts ...audio.StemSeparationOption) (*audio.StemSeparationResult, error) { + req = req.Apply(opts...) + if len(req.Audio) == 0 { + return nil, fmt.Errorf("%w: stem separation requires audio bytes", llm.ErrUnsupported) + } + mode := strings.ToLower(strings.TrimSpace(req.Mode)) + if mode != "" && mode != "two" && mode != "four" { + return nil, fmt.Errorf("%w: stem mode must be \"two\" or \"four\", got %q", llm.ErrUnsupported, req.Mode) + } + format := strings.ToLower(strings.TrimSpace(req.Format)) + if format != "" && format != "mp3" && format != "wav" { + return nil, fmt.Errorf("%w: stem format must be \"mp3\" or \"wav\", got %q", llm.ErrUnsupported, req.Format) + } + upPath, err := upstreamPath(m.id, "/v1/stems") + if err != nil { + return nil, err + } + // Demucs' two-stem mode is "isolate one source vs the rest"; vocals is + // the split this surface promises ("two" = vocals + accompaniment). + twoStems := "" + if mode == "two" { + twoStems = "vocals" + } + body, contentType, err := buildMultipart("build stems form", + filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio}, + []formField{ + {"model", req.Model, false}, + {"two_stems", twoStems, false}, + {"format", format, false}, + }) + if err != nil { + return nil, err + } + raw, respType, err := m.p.doRaw(ctx, http.MethodPost, upPath, m.id, contentType, body, maxStemsResponseBytes) + if err != nil { + return nil, err + } + if len(raw) == 0 { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems response contained no data"} + } + zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw))) + if err != nil { + // A non-zip 2xx body is a misconfigured upstream (an HTML error page + // behind a proxy, a JSON soft error) — fail loud, quoting a slice. + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, + Message: fmt.Sprintf("stems response is not a zip (Content-Type %q): %s", respType, truncateForError(raw))} + } + res := &audio.StemSeparationResult{} + for _, f := range zr.File { + if f.FileInfo().IsDir() { + continue + } + data, err := readZipEntry(f) + if err != nil { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, + Message: fmt.Sprintf("stems zip entry %q: %v", f.Name, err)} + } + // Entry name → stem name; extension → MIME. Entries may sit in a + // per-model directory ("htdemucs/vocals.mp3"), so use the base name. + base := path.Base(f.Name) + ext := path.Ext(base) + res.Stems = append(res.Stems, audio.Stem{ + Name: strings.TrimSuffix(base, ext), + Audio: data, + MIME: stemMIME(ext), + }) + } + if len(res.Stems) == 0 { + return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems zip contained no stems"} + } + return res, nil +} + +// readZipEntry decompresses one entry, bounded by maxStemEntryBytes so a +// zip bomb can't allocate without limit. +func readZipEntry(f *zip.File) ([]byte, error) { + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() + data, err := io.ReadAll(io.LimitReader(rc, maxStemEntryBytes+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > maxStemEntryBytes { + return nil, fmt.Errorf("decompressed entry exceeds %d bytes", int64(maxStemEntryBytes)) + } + return data, nil +} + +// stemMIME maps a stem file extension to its MIME type, reusing speechMIME's +// format table ("" and unknown extensions land on the mp3 default — Demucs' +// own default container). +func stemMIME(ext string) string { + return speechMIME("", strings.TrimPrefix(strings.ToLower(ext), ".")) +} diff --git a/provider/llamaswap/stems_sfx_enhance_test.go b/provider/llamaswap/stems_sfx_enhance_test.go new file mode 100644 index 0000000..41e93bd --- /dev/null +++ b/provider/llamaswap/stems_sfx_enhance_test.go @@ -0,0 +1,332 @@ +package llamaswap + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "gitea.stevedudenhoeffer.com/steve/majordomo/audio" + "gitea.stevedudenhoeffer.com/steve/majordomo/llm" + "gitea.stevedudenhoeffer.com/steve/majordomo/musicgen" +) + +// stemsZipFixture builds a Demucs-style stems zip: entries under a per-model +// directory, entry name = stem name, extension = container. +func stemsZipFixture(t *testing.T, entries map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, data := range entries { + w, err := zw.Create(name) + if err != nil { + t.Fatalf("zip create: %v", err) + } + if _, err := w.Write([]byte(data)); err != nil { + t.Fatalf("zip write: %v", err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("zip close: %v", err) + } + return buf.Bytes() +} + +func TestSeparateStems(t *testing.T) { + zipBody := stemsZipFixture(t, map[string]string{ + "htdemucs/vocals.mp3": "VOX", + "htdemucs/no_vocals.mp3": "ACC", + }) + var gotPath, gotTwoStems, gotModel, gotFormat, gotFilename string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + gotTwoStems = r.FormValue("two_stems") + gotModel = r.FormValue("model") + gotFormat = r.FormValue("format") + if _, hdr, err := r.FormFile("file"); err == nil { + gotFilename = hdr.Filename + } else { + t.Errorf("file part: %v", err) + } + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(zipBody) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ss, err := p.StemSeparatorModel("audioutils") + if err != nil { + t.Fatalf("StemSeparatorModel: %v", err) + } + res, err := ss.SeparateStems(context.Background(), + audio.StemSeparationRequest{Audio: []byte("SONG"), MIME: "audio/mpeg"}, + audio.WithStemMode("two"), audio.WithStemModel("htdemucs_ft"), audio.WithStemFormat("mp3")) + if err != nil { + t.Fatalf("SeparateStems: %v", err) + } + if gotPath != "/upstream/audioutils/v1/stems" { + t.Errorf("path = %q", gotPath) + } + if gotTwoStems != "vocals" || gotModel != "htdemucs_ft" || gotFormat != "mp3" || gotFilename != "audio.mp3" { + t.Errorf("two_stems/model/format/filename = %q/%q/%q/%q", gotTwoStems, gotModel, gotFormat, gotFilename) + } + if len(res.Stems) != 2 { + t.Fatalf("stems = %+v", res.Stems) + } + byName := map[string]audio.Stem{} + for _, s := range res.Stems { + byName[s.Name] = s + } + if v := byName["vocals"]; string(v.Audio) != "VOX" || v.MIME != "audio/mpeg" { + t.Errorf("vocals = %+v", v) + } + if a := byName["no_vocals"]; string(a.Audio) != "ACC" || a.MIME != "audio/mpeg" { + t.Errorf("no_vocals = %+v", a) + } +} + +func TestSeparateStemsOmitsUnsetFields(t *testing.T) { + zipBody := stemsZipFixture(t, map[string]string{"vocals.wav": "V"}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + for _, k := range []string{"two_stems", "model", "format"} { + if v, ok := r.MultipartForm.Value[k]; ok { + t.Errorf("unset request sent %q = %v, want omitted", k, v) + } + } + _, _ = w.Write(zipBody) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ss, _ := p.StemSeparatorModel("audioutils") + res, err := ss.SeparateStems(context.Background(), + audio.StemSeparationRequest{Audio: []byte("SONG")}) + if err != nil { + t.Fatalf("SeparateStems: %v", err) + } + // Top-level entry, wav extension. + if len(res.Stems) != 1 || res.Stems[0].Name != "vocals" || res.Stems[0].MIME != "audio/wav" { + t.Errorf("stems = %+v", res.Stems) + } +} + +func TestSeparateStemsRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + ss, _ := p.StemSeparatorModel("audioutils") + if _, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("no audio: err = %v, want ErrUnsupported", err) + } + if _, err := ss.SeparateStems(context.Background(), + audio.StemSeparationRequest{Audio: []byte{1}, Mode: "three"}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("mode three: err = %v, want ErrUnsupported", err) + } + if _, err := ss.SeparateStems(context.Background(), + audio.StemSeparationRequest{Audio: []byte{1}, Format: "flac"}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("format flac: err = %v, want ErrUnsupported", err) + } +} + +func TestSeparateStemsRejectsNonZip(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("proxy error page")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ss, _ := p.StemSeparatorModel("audioutils") + _, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for non-zip body", err) + } +} + +func TestSeparateStemsRejectsEmptyZip(t *testing.T) { + zipBody := stemsZipFixture(t, map[string]string{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(zipBody) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + ss, _ := p.StemSeparatorModel("audioutils") + _, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for stem-less zip", err) + } +} + +// wavFixture is a minimal RIFF/WAVE header so http.DetectContentType sniffs +// audio/wave. +func wavFixture() []byte { + return []byte("RIFF\x24\x00\x00\x00WAVEfmt ") +} + +func TestSFXGenerate(t *testing.T) { + var gotPath string + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + _ = json.NewDecoder(r.Body).Decode(&gotBody) + w.Header().Set("Content-Type", "audio/wav") + _, _ = w.Write(wavFixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + sm, err := p.SFXModel("sfxgen-stableaudio") + if err != nil { + t.Fatalf("SFXModel: %v", err) + } + res, err := sm.Generate(context.Background(), + musicgen.Request{Prompt: "glass shattering", DurationSeconds: 8}, + musicgen.WithSteps(50), musicgen.WithCFGScale(7), musicgen.WithSeed(42)) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if gotPath != "/upstream/sfxgen-stableaudio/v1/sfx" { + t.Errorf("path = %q", gotPath) + } + want := map[string]any{"prompt": "glass shattering", "seconds": 8.0, "steps": 50.0, "cfg_scale": 7.0, "seed": 42.0} + for k, w := range want { + if gotBody[k] != w { + t.Errorf("%s = %v, want %v", k, gotBody[k], w) + } + } + if res.Audio.MIME != "audio/wav" || len(res.Audio.Data) == 0 { + t.Errorf("audio = %q/%d bytes", res.Audio.MIME, len(res.Audio.Data)) + } +} + +func TestSFXOmitsDefaults(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&gotBody) + // No Content-Type: the RIFF sniff must still label the clip. + w.Header()["Content-Type"] = nil + _, _ = w.Write(wavFixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + sm, _ := p.SFXModel("sfxgen-stableaudio") + res, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"}) + if err != nil { + t.Fatalf("Generate: %v", err) + } + for _, k := range []string{"seconds", "steps", "cfg_scale", "seed"} { + if v, ok := gotBody[k]; ok { + t.Errorf("unset request sent %q = %v, want omitted", k, v) + } + } + if res.Audio.MIME != "audio/wav" { + t.Errorf("MIME = %q, want sniffed audio/wav", res.Audio.MIME) + } +} + +func TestSFXRejectsBadArgs(t *testing.T) { + p := New(WithBaseURL("http://unused")) + sm, _ := p.SFXModel("sfxgen-stableaudio") + if _, err := sm.Generate(context.Background(), musicgen.Request{Prompt: " "}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("empty prompt: err = %v, want ErrUnsupported", err) + } + if _, err := sm.Generate(context.Background(), + musicgen.Request{Prompt: "boom", Lyrics: "la la"}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("lyrics: err = %v, want ErrUnsupported", err) + } + if _, err := sm.Generate(context.Background(), + musicgen.Request{Prompt: "boom", Format: "mp3"}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("format mp3: err = %v, want ErrUnsupported", err) + } +} + +func TestSFXRejectsNonAudioResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"detail":"queue full"}`)) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + sm, _ := p.SFXModel("sfxgen-stableaudio") + _, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for non-audio body", err) + } +} + +func TestEnhance(t *testing.T) { + var gotPath, gotFilename string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Fatalf("parse multipart: %v", err) + } + if _, hdr, err := r.FormFile("file"); err == nil { + gotFilename = hdr.Filename + } else { + t.Errorf("file part: %v", err) + } + w.Header().Set("Content-Type", "audio/wav") + _, _ = w.Write(wavFixture()) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + en, err := p.SpeechEnhancerModel("audioutils") + if err != nil { + t.Fatalf("SpeechEnhancerModel: %v", err) + } + res, err := en.Enhance(context.Background(), + audio.EnhancementRequest{Audio: []byte("NOISY"), MIME: "audio/ogg"}) + if err != nil { + t.Fatalf("Enhance: %v", err) + } + if gotPath != "/upstream/audioutils/v1/enhance" { + t.Errorf("path = %q", gotPath) + } + if gotFilename != "audio.ogg" { + t.Errorf("filename = %q", gotFilename) + } + if res.MIME != "audio/wav" || len(res.Audio) == 0 { + t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio)) + } +} + +func TestEnhanceRejectsEmptyAudio(t *testing.T) { + p := New(WithBaseURL("http://unused")) + en, _ := p.SpeechEnhancerModel("audioutils") + if _, err := en.Enhance(context.Background(), audio.EnhancementRequest{}); !errors.Is(err, llm.ErrUnsupported) { + t.Errorf("err = %v, want ErrUnsupported", err) + } +} + +func TestEnhanceRejectsNonAudioResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("oops")) + })) + defer srv.Close() + + p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client())) + en, _ := p.SpeechEnhancerModel("audioutils") + _, err := en.Enhance(context.Background(), audio.EnhancementRequest{Audio: []byte("NOISY")}) + var apiErr *llm.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("err = %v, want APIError for non-audio body", err) + } +}