feat: wave-3 audio surfaces — stems, sfx, speech enhance, voice clone, translate (ADR-0024)
CI / Tidy (pull_request) Successful in 9m27s
CI / Build & Test (pull_request) Successful in 10m37s
Gadfly review (reusable) / review (pull_request) Successful in 41m14s
Adversarial Review (Gadfly) / review (pull_request) Successful in 41m14s

- audio.StemSeparator/StemSeparationProvider: Demucs zip transport via
  POST /upstream/<id>/v1/stems (Mode two -> two_stems=vocals; model +
  format fields); bounded zip unpack, entry name -> stem, ext -> MIME.
- SFXModel reuses musicgen against the sync /upstream/<id>/v1/sfx route
  (JSON prompt/seconds/steps/cfg_scale/seed -> WAV); musicgen.Request
  gains CFGScale.
- audio.SpeechEnhancer/SpeechEnhancementProvider:
  POST /upstream/<id>/v1/enhance -> WAV (result reuses SpeechResult).
- SpeechRequest.ReferenceAudio/ReferenceMIME (+WithReferenceAudio):
  llamaswap switches to the chatterbox clone route
  POST /upstream/<id>/v1/audio/speech/upload (input + voice_file),
  wav MIME fallback.
- TranscriptionRequest.Translate (+WithTranslate): translate=true form
  field, language=auto forced when no explicit hint (whisper.cpp default
  en would skip translation).
- httptest contract tests (zip unpack, clone-route switch, translate +
  auto-language injection); ADR-0024 (index row deferred — MJ-A backfills
  the ADR index table and parallel edits would conflict).

Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
2026-07-16 17:01:11 -04:00
co-authored by Claude Fable 5
parent cd43009672
commit b3a172a053
11 changed files with 1182 additions and 2 deletions
+28
View File
@@ -30,6 +30,17 @@ type SpeechRequest struct {
// Speed is the playback-rate multiplier; 0 = backend default (1.0).
Speed float64
// ReferenceAudio is a short voice sample for zero-shot voice cloning
// (chatterbox style): when set, the model speaks Input in the sampled
// voice instead of a named Voice. nil = normal synthesis. Backends
// without cloning support must reject a reference-carrying request
// rather than silently ignoring it.
ReferenceAudio []byte
// ReferenceMIME is the reference audio's MIME type (e.g. "audio/wav");
// "" = let the backend sniff it.
ReferenceMIME string
}
// SpeechResult is the canonical synthesis result: raw audio bytes plus the
@@ -59,6 +70,11 @@ func WithFormat(f string) SpeechOption { return func(r *SpeechRequest) { r.Forma
// WithSpeed sets the playback-rate multiplier.
func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } }
// WithReferenceAudio provides a voice sample for zero-shot voice cloning.
func WithReferenceAudio(data []byte, mime string) SpeechOption {
return func(r *SpeechRequest) { r.ReferenceAudio, r.ReferenceMIME = data, mime }
}
// Apply returns a copy of the request with all options applied. Providers
// call this once at the top of Speak.
func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest {
@@ -121,6 +137,13 @@ type TranscriptionRequest struct {
// Prompt is optional context or vocabulary to bias decoding; "" = none.
Prompt string
// Translate requests an English translation of the speech instead of a
// same-language transcript (whisper's translate task). When set and no
// Language is given, providers must force source-language auto-detection
// — a backend whose default language is "en" would otherwise skip
// translation entirely.
Translate bool
}
// TranscriptionResult is the canonical transcription result.
@@ -145,6 +168,11 @@ func WithPrompt(p string) TranscriptionOption {
return func(r *TranscriptionRequest) { r.Prompt = p }
}
// WithTranslate requests an English translation instead of a transcript.
func WithTranslate() TranscriptionOption {
return func(r *TranscriptionRequest) { r.Translate = true }
}
// Apply returns a copy of the request with all options applied.
func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest {
for _, opt := range opts {
+66
View File
@@ -0,0 +1,66 @@
package audio
import "context"
// EnhancementRequest asks a speech-enhancement backend (DeepFilterNet style)
// to denoise a recording. Audio is carried as bytes (never a URL), mirroring
// TranscriptionRequest (ADR-0024).
type EnhancementRequest struct {
// Audio is the encoded audio to enhance.
Audio []byte
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
// sniff it.
MIME string
// Filename is the multipart filename hint some backends key their format
// detection on; "" derives one from MIME or falls back to "audio".
Filename string
}
// EnhancementOption mutates an EnhancementRequest before it is sent.
// Reserved for future request settings (the reference backend takes no
// parameters).
type EnhancementOption func(*EnhancementRequest)
// Apply returns a copy of the request with all options applied.
func (r EnhancementRequest) Apply(opts ...EnhancementOption) EnhancementRequest {
for _, opt := range opts {
opt(&r)
}
return r
}
// SpeechEnhancer denoises speech recordings. The result reuses SpeechResult
// (audio bytes + MIME + Raw) — enhancement is audio-in/audio-out exactly
// like synthesis.
type SpeechEnhancer interface {
// Enhance returns the denoised audio.
Enhance(ctx context.Context, req EnhancementRequest, opts ...EnhancementOption) (*SpeechResult, error)
}
// SpeechEnhancerModelOption configures a SpeechEnhancer at construction time.
// Reserved for future per-model settings.
type SpeechEnhancerModelOption func(*SpeechEnhancerModelConfig)
// SpeechEnhancerModelConfig carries per-model construction settings.
type SpeechEnhancerModelConfig struct{}
// ApplySpeechEnhancerModelOptions folds options into a config.
func ApplySpeechEnhancerModelOptions(opts []SpeechEnhancerModelOption) SpeechEnhancerModelConfig {
var cfg SpeechEnhancerModelConfig
for _, opt := range opts {
opt(&cfg)
}
return cfg
}
// SpeechEnhancementProvider mints SpeechEnhancers bound to one backend.
type SpeechEnhancementProvider interface {
// Name is the registry identifier for the provider.
Name() string
// SpeechEnhancerModel returns a SpeechEnhancer bound to the given id
// (passed through to the backend verbatim; no catalog validation).
SpeechEnhancerModel(id string, opts ...SpeechEnhancerModelOption) (SpeechEnhancer, error)
}
+113
View File
@@ -0,0 +1,113 @@
package audio
import "context"
// StemSeparationRequest asks a source-separation backend (Demucs style) to
// split a mix into stems. Audio is carried as bytes (never a URL), mirroring
// TranscriptionRequest. Zero values mean "backend default" (ADR-0024).
type StemSeparationRequest struct {
// Audio is the encoded mix to separate.
Audio []byte
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
// sniff it.
MIME string
// Filename is the multipart filename hint some backends key their format
// detection on; "" derives one from MIME or falls back to "audio".
Filename string
// Mode selects the split: "two" (vocals + accompaniment) or "four"
// (vocals/drums/bass/other); "" = backend default (four).
Mode string
// Model selects the separator's internal network where the backend
// offers several (Demucs: "htdemucs", "htdemucs_ft"); "" = backend
// default. This is NOT the provider model id — that is fixed when the
// StemSeparator is minted (mirrors BackgroundRemovalRequest.Net).
Model string
// Format is the per-stem audio container ("mp3" or "wav");
// "" = backend default.
Format string
}
// Stem is one separated source.
type Stem struct {
// Name is the stem's name ("vocals", "drums", "bass", "other",
// "no_vocals", ...), taken from the backend's own labelling.
Name string
// Audio is the encoded stem.
Audio []byte
// MIME is the stem's audio MIME type, e.g. "audio/mpeg".
MIME string
}
// StemSeparationResult is the canonical separation result.
type StemSeparationResult struct {
// Stems are the separated sources, in the order the backend returned
// them.
Stems []Stem
}
// StemSeparationOption mutates a StemSeparationRequest before it is sent.
type StemSeparationOption func(*StemSeparationRequest)
// WithStemMode selects the split ("two" or "four").
func WithStemMode(m string) StemSeparationOption {
return func(r *StemSeparationRequest) { r.Mode = m }
}
// WithStemModel selects the separator's internal network.
func WithStemModel(m string) StemSeparationOption {
return func(r *StemSeparationRequest) { r.Model = m }
}
// WithStemFormat sets the per-stem audio container ("mp3", "wav").
func WithStemFormat(f string) StemSeparationOption {
return func(r *StemSeparationRequest) { r.Format = f }
}
// Apply returns a copy of the request with all options applied.
func (r StemSeparationRequest) Apply(opts ...StemSeparationOption) StemSeparationRequest {
for _, opt := range opts {
opt(&r)
}
return r
}
// StemSeparator splits a mix into stems.
type StemSeparator interface {
// SeparateStems returns the separated sources. Separation is CPU-bound
// and slow (minutes for a full song); bound the call with a context
// deadline.
SeparateStems(ctx context.Context, req StemSeparationRequest, opts ...StemSeparationOption) (*StemSeparationResult, error)
}
// StemSeparatorModelOption configures a StemSeparator at construction time.
// Reserved for future per-model settings.
type StemSeparatorModelOption func(*StemSeparatorModelConfig)
// StemSeparatorModelConfig carries per-model construction settings.
type StemSeparatorModelConfig struct{}
// ApplyStemSeparatorModelOptions folds options into a config.
func ApplyStemSeparatorModelOptions(opts []StemSeparatorModelOption) StemSeparatorModelConfig {
var cfg StemSeparatorModelConfig
for _, opt := range opts {
opt(&cfg)
}
return cfg
}
// StemSeparationProvider mints StemSeparators bound to one backend.
type StemSeparationProvider interface {
// Name is the registry identifier for the provider.
Name() string
// StemSeparatorModel returns a StemSeparator bound to the given id
// (passed through to the backend verbatim; no catalog validation).
StemSeparatorModel(id string, opts ...StemSeparatorModelOption) (StemSeparator, error)
}
+65
View File
@@ -0,0 +1,65 @@
# ADR-0024: Wave-3 audio surfaces (stems, SFX, speech enhance, voice clone, translate)
Status: Accepted (2026-07-16)
## Context
The llama-swap host is gaining wave-3 audio capabilities (spec: mort
docs/specs/2026-07-16-llamaswap-wave3.md): Demucs stem separation and
DeepFilterNet speech enhancement (both on the CPU `audioutils` shim), Stable
Audio Open sound effects (`sfxgen`), plus two upgrades to existing models —
chatterbox's stateless voice-clone route and whisper.cpp's per-request
translate flag (both verified against the live images 2026-07-16).
## Decision
1. **`audio.StemSeparator` / `StemSeparationProvider`** —
`StemSeparationRequest{Audio, MIME, Filename, Mode, Model, Format}`
`StemSeparationResult{Stems []Stem{Name, Audio, MIME}}`.
- `Mode` is the caller-facing split: `"two"` (vocals + accompaniment,
sent as Demucs' `two_stems=vocals`) or `"four"`; `""` = backend
default. `Model` selects the Demucs variant (`htdemucs`/`htdemucs_ft`),
mirroring `BackgroundRemovalRequest.Net`.
- **The wire format is a ZIP** (`POST /upstream/<id>/v1/stems`): four WAV
stems would blow any JSON-of-base64 budget. Entry name → stem name,
extension → MIME; entries may sit under a per-model directory. Unpacking
is bounded per entry (zip-bomb guard) and a non-zip 2xx body fails loud.
2. **`SFXModel` reuses `musicgen`** — a sound effect is a short audio clip
from a text prompt; only the provider method differs. The sfxgen route
(`POST /upstream/<id>/v1/sfx`, JSON `{prompt, seconds, steps?, cfg_scale?,
seed?}`) is SYNCHRONOUS (WAV body), unlike ACE-Step's job queue.
`musicgen.Request` gains `CFGScale *float64` for it; lyrics and non-wav
formats are rejected (the model can't honor them). The ~11s model ceiling
is the backend's to enforce.
3. **`audio.SpeechEnhancer` / `SpeechEnhancementProvider`** —
`EnhancementRequest{Audio, MIME, Filename}`
`POST /upstream/<id>/v1/enhance` → WAV. The result reuses `SpeechResult`;
audio-in/audio-out needs no new result type.
4. **Voice cloning is a `SpeechRequest` field, not a new surface**
`ReferenceAudio []byte` + `ReferenceMIME`. When set, the llamaswap
speech model switches from JSON `/v1/audio/speech` to multipart
`POST /upstream/<id>/v1/audio/speech/upload` (fields `input` +
`voice_file`; verified live: stateless per-request cloning, no voice
library). Voice/format/speed still ride when set; the clone route's MIME
fallback is wav (not the JSON route's mp3). Backends without cloning must
reject a reference-carrying request rather than silently ignore it.
5. **Translation is a `TranscriptionRequest` bool**`Translate` maps to
whisper.cpp's `translate=true` form field (server.cpp:534). Because that
server's language DEFAULT is `en` (which silently skips translation), the
provider forces `language=auto` when translating without an explicit
language hint; an explicit hint wins.
6. **Binary success bodies are validated before wrapping** (ADR-0020 rule):
sfx/enhance require positive evidence of audio-ness (declared audio/*
Content-Type or sniffed RIFF/WAVE), stems require a parseable zip with
at least one stem.
## Consequences
- `audio` grows from three surfaces to five; the SFX surface adds zero new
types (musicgen reuse) — one format→MIME table and one multipart builder
keep serving every audio endpoint.
- The clone-route switch means one `SpeechModel` can answer over two wire
shapes; tests pin both routes so a regression can't silently drop cloning.
- `two_stems` is hardwired to vocals: "isolate X vs the rest" for other
sources is a backend capability not exposed in v1 (add a field when a
caller needs it, not before).
+8
View File
@@ -40,6 +40,11 @@ type Request struct {
// Steps is the number of inference steps; nil = backend default.
Steps *int
// CFGScale is the classifier-free-guidance scale; nil = backend default.
// Architecture-sensitive, so prefer leaving it nil unless the caller
// knows the target model. Backends without the knob ignore it.
CFGScale *float64
// Seed fixes the RNG seed for reproducible output; nil = backend
// default (random).
Seed *int64
@@ -73,6 +78,9 @@ func WithFormat(f string) Option { return func(r *Request) { r.Format = f } }
// WithSteps overrides the number of inference steps.
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }
// WithCFGScale overrides the classifier-free-guidance scale.
func WithCFGScale(s float64) Option { return func(r *Request) { r.CFGScale = &s } }
// WithSeed fixes the RNG seed.
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }
+66 -2
View File
@@ -9,6 +9,7 @@ import (
"mime"
"net/http"
"net/url"
"strconv"
"strings"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
@@ -42,7 +43,11 @@ type speechRequest struct {
Speed float64 `json:"speed,omitempty"`
}
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech.
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech, or —
// when the request carries reference audio for voice cloning — via multipart
// POST {base}/upstream/<id>/v1/audio/speech/upload (the chatterbox clone
// route; verified live 2026-07-16: stateless per-request cloning, fields
// `input` + `voice_file`).
func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) {
req = req.Apply(opts...)
if strings.TrimSpace(req.Input) == "" {
@@ -51,6 +56,9 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
if req.Speed < 0 {
return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed)
}
if len(req.ReferenceAudio) > 0 {
return m.speakWithReference(ctx, req)
}
wire := speechRequest{
Model: m.id,
Input: req.Input,
@@ -72,6 +80,49 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil
}
// speakWithReference performs zero-shot voice cloning via the upstream
// passthrough clone route. The reference sample rides as the `voice_file`
// part and the text as an `input` field; voice/format/speed still go on the
// wire when set (upstreams ignore fields they don't understand). The route's
// default container is WAV, so MIME resolution falls back to audio/wav (not
// the JSON route's mp3) when neither the response header nor the requested
// format decides.
func (m *speechModel) speakWithReference(ctx context.Context, req audio.SpeechRequest) (*audio.SpeechResult, error) {
path, err := upstreamPath(m.id, "/v1/audio/speech/upload")
if err != nil {
return nil, err
}
speed := ""
if req.Speed != 0 {
speed = strconv.FormatFloat(req.Speed, 'g', -1, 64)
}
body, formType, err := buildMultipart("build speech clone form",
filePart{field: "voice_file", filename: transcriptionFilename("", req.ReferenceMIME), data: req.ReferenceAudio},
[]formField{
{"input", req.Input, true},
{"voice", req.Voice, false},
{"response_format", req.Format, false},
{"speed", speed, false},
})
if err != nil {
return nil, err
}
audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, formType, body, maxResponseBytes)
if err != nil {
return nil, err
}
if len(audioBytes) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech clone response contained no audio"}
}
mimeType := "audio/wav"
if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
mimeType = mt
} else if req.Format != "" {
mimeType = speechMIME("", req.Format)
}
return &audio.SpeechResult{Audio: audioBytes, MIME: mimeType}, nil
}
// speechMIME resolves the result MIME type: the response Content-Type when it
// is a concrete audio type, else a mapping from the requested format, else
// audio/mpeg (the OpenAI endpoint's default container is mp3).
@@ -118,13 +169,26 @@ func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.Transcrip
return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported)
}
// Translation is a per-request bool form field on whisper.cpp's server
// (server.cpp:534, verified 2026-07-16). Its language DEFAULT is "en",
// which would make translate a no-op — so an explicit language hint
// wins, but an unset one is forced to auto-detection.
translate := ""
language := req.Language
if req.Translate {
translate = "true"
if language == "" {
language = "auto"
}
}
buf, formType, err := buildMultipart("build transcription form",
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
[]formField{
{"model", m.id, true},
{"response_format", "json", true},
{"language", req.Language, false},
{"language", language, false},
{"prompt", req.Prompt, false},
{"translate", translate, false},
})
if err != nil {
return nil, err
+179
View File
@@ -0,0 +1,179 @@
package llamaswap
import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
)
func TestSpeakWithReferenceUsesCloneRoute(t *testing.T) {
var gotPath, gotInput, gotVoice, gotFilename string
var gotRef []byte
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotInput = r.FormValue("input")
gotVoice = r.FormValue("voice")
f, hdr, err := r.FormFile("voice_file")
if err != nil {
t.Fatalf("voice_file part: %v", err)
}
defer f.Close()
gotRef, _ = io.ReadAll(f)
gotFilename = hdr.Filename
w.Header().Set("Content-Type", "audio/wav")
_, _ = w.Write(wavFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SpeechModel("chatterbox")
res, err := sm.Speak(context.Background(),
audio.SpeechRequest{Input: "hello in my voice", Voice: "narrator"},
audio.WithReferenceAudio([]byte("REFWAV"), "audio/wav"))
if err != nil {
t.Fatalf("Speak: %v", err)
}
if gotPath != "/upstream/chatterbox/v1/audio/speech/upload" {
t.Errorf("path = %q, want clone route", gotPath)
}
if gotInput != "hello in my voice" || gotVoice != "narrator" {
t.Errorf("input/voice = %q/%q", gotInput, gotVoice)
}
if string(gotRef) != "REFWAV" || gotFilename != "audio.wav" {
t.Errorf("ref = %q name = %q", gotRef, gotFilename)
}
if res.MIME != "audio/wav" || len(res.Audio) == 0 {
t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio))
}
}
func TestSpeakWithReferenceDefaultsToWavMIME(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
for _, k := range []string{"voice", "response_format", "speed"} {
if v, ok := r.MultipartForm.Value[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
w.Header()["Content-Type"] = nil // no declared type at all
_, _ = w.Write([]byte("RAWAUDIO"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SpeechModel("chatterbox")
res, err := sm.Speak(context.Background(),
audio.SpeechRequest{Input: "hi"},
audio.WithReferenceAudio([]byte("REF"), ""))
if err != nil {
t.Fatalf("Speak: %v", err)
}
// The clone route's default container is wav, not the JSON route's mp3.
if res.MIME != "audio/wav" {
t.Errorf("MIME = %q, want audio/wav fallback", res.MIME)
}
}
func TestSpeakWithoutReferenceKeepsJSONRoute(t *testing.T) {
var gotPath, gotContentType string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
gotContentType = r.Header.Get("Content-Type")
_, _ = w.Write([]byte("MP3"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SpeechModel("chatterbox")
if _, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: "hi"}); err != nil {
t.Fatalf("Speak: %v", err)
}
if gotPath != "/v1/audio/speech" || gotContentType != "application/json" {
t.Errorf("path/content-type = %q/%q, want JSON route", gotPath, gotContentType)
}
}
func TestTranscribeTranslateForcesAutoLanguage(t *testing.T) {
var gotTranslate, gotLanguage string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotTranslate = r.FormValue("translate")
gotLanguage = r.FormValue("language")
_, _ = w.Write([]byte(`{"text":"hello"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
res, err := tm.Transcribe(context.Background(),
audio.TranscriptionRequest{Audio: []byte("AUDIO")},
audio.WithTranslate())
if err != nil {
t.Fatalf("Transcribe: %v", err)
}
// whisper.cpp's server default language is "en", which would skip
// translation — translate must force auto-detection when no explicit
// language hint was given.
if gotTranslate != "true" || gotLanguage != "auto" {
t.Errorf("translate/language = %q/%q, want true/auto", gotTranslate, gotLanguage)
}
if res.Text != "hello" {
t.Errorf("text = %q", res.Text)
}
}
func TestTranscribeTranslateKeepsExplicitLanguage(t *testing.T) {
var gotTranslate, gotLanguage string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotTranslate = r.FormValue("translate")
gotLanguage = r.FormValue("language")
_, _ = w.Write([]byte(`{"text":"hello"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
if _, err := tm.Transcribe(context.Background(),
audio.TranscriptionRequest{Audio: []byte("AUDIO"), Language: "de", Translate: true}); err != nil {
t.Fatalf("Transcribe: %v", err)
}
if gotTranslate != "true" || gotLanguage != "de" {
t.Errorf("translate/language = %q/%q, want true/de", gotTranslate, gotLanguage)
}
}
func TestTranscribeWithoutTranslateOmitsFields(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
for _, k := range []string{"translate", "language"} {
if v, ok := r.MultipartForm.Value[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
_, _ = w.Write([]byte(`{"text":"hi"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
if _, err := tm.Transcribe(context.Background(),
audio.TranscriptionRequest{Audio: []byte("AUDIO")}); err != nil {
t.Fatalf("Transcribe: %v", err)
}
}
+61
View File
@@ -0,0 +1,61 @@
// enhance.go implements audio.SpeechEnhancementProvider against a
// DeepFilterNet shim (audioutils) reached through llama-swap's /upstream
// passthrough (ADR-0024):
//
// POST /upstream/<id>/v1/enhance multipart file -> WAV
package llamaswap
import (
"context"
"fmt"
"net/http"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)
// SpeechEnhancerModel implements audio.SpeechEnhancementProvider. The id
// selects which upstream llama-swap loads.
func (p *Provider) SpeechEnhancerModel(id string, opts ...audio.SpeechEnhancerModelOption) (audio.SpeechEnhancer, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = audio.ApplySpeechEnhancerModelOptions(opts)
return &speechEnhancerModel{p: p, id: id}, nil
}
type speechEnhancerModel struct {
p *Provider
id string
}
// Enhance implements audio.SpeechEnhancer. The endpoint always answers WAV.
func (m *speechEnhancerModel) Enhance(ctx context.Context, req audio.EnhancementRequest, opts ...audio.EnhancementOption) (*audio.SpeechResult, error) {
req = req.Apply(opts...)
if len(req.Audio) == 0 {
return nil, fmt.Errorf("%w: speech enhancement requires audio bytes", llm.ErrUnsupported)
}
path, err := upstreamPath(m.id, "/v1/enhance")
if err != nil {
return nil, err
}
body, contentType, err := buildMultipart("build enhance form",
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
nil)
if err != nil {
return nil, err
}
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "enhance response contained no audio"}
}
mimeType := audioResultMIME(respType, raw)
if mimeType == "" {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("enhance response is not audio (Content-Type %q): %s", respType, truncateForError(raw))}
}
return &audio.SpeechResult{Audio: raw, MIME: mimeType}, nil
}
+118
View File
@@ -0,0 +1,118 @@
// sfx.go implements a second musicgen.Model factory against a Stable Audio
// Open shim (sfxgen) reached through llama-swap's /upstream passthrough
// (ADR-0024):
//
// POST /upstream/<id>/v1/sfx JSON {prompt, seconds?, steps?, cfg_scale?, seed?}
//
// Unlike the ACE-Step music path (music.go), /v1/sfx is SYNCHRONOUS: the
// response body is the finished WAV clip — no job queue, no polling. The
// surface reuses the musicgen types (a sound effect is a short audio clip
// from a text prompt); only the provider method differs.
package llamaswap
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
)
// SFXModel returns a musicgen.Model bound to a sync sound-effect backend.
// The id selects which upstream llama-swap loads.
func (p *Provider) SFXModel(id string, opts ...musicgen.ModelOption) (musicgen.Model, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = musicgen.ApplyModelOptions(opts)
return &sfxModel{p: p, id: id}, nil
}
type sfxModel struct {
p *Provider
id string
}
// sfxRequest is the sfxgen shim's /v1/sfx shape. Optional fields stay off
// the wire so the model's own defaults apply.
type sfxRequest struct {
Prompt string `json:"prompt"`
Seconds int `json:"seconds,omitempty"`
Steps *int `json:"steps,omitempty"`
CFGScale *float64 `json:"cfg_scale,omitempty"`
Seed *int64 `json:"seed,omitempty"`
}
// Generate implements musicgen.Model. The clip-length ceiling (~11s for
// Stable Audio Open Small) is the backend's to enforce, not this client's.
func (m *sfxModel) Generate(ctx context.Context, req musicgen.Request, opts ...musicgen.Option) (*musicgen.Result, error) {
req = req.Apply(opts...)
if strings.TrimSpace(req.Prompt) == "" {
return nil, fmt.Errorf("%w: sfx generation requires a prompt", llm.ErrUnsupported)
}
if req.Lyrics != "" {
return nil, fmt.Errorf("%w: sfx generation does not support lyrics", llm.ErrUnsupported)
}
if req.DurationSeconds < 0 {
return nil, fmt.Errorf("%w: duration must be >= 0, got %d", llm.ErrUnsupported, req.DurationSeconds)
}
if req.Steps != nil && *req.Steps <= 0 {
return nil, fmt.Errorf("%w: inference steps must be > 0, got %d", llm.ErrUnsupported, *req.Steps)
}
// The endpoint emits WAV only; a caller asking for another container
// would silently get mislabelled bytes — reject instead.
if f := strings.ToLower(strings.TrimSpace(req.Format)); f != "" && f != "wav" {
return nil, fmt.Errorf("%w: sfx output is wav only, got format %q", llm.ErrUnsupported, req.Format)
}
path, err := upstreamPath(m.id, "/v1/sfx")
if err != nil {
return nil, err
}
wire := sfxRequest{
Prompt: req.Prompt,
Seconds: req.DurationSeconds,
Steps: req.Steps,
CFGScale: req.CFGScale,
Seed: req.Seed,
}
encoded, err := json.Marshal(wire)
if err != nil {
return nil, fmt.Errorf("llama-swap: encode sfx request: %w", err)
}
raw, contentType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, "application/json", bytes.NewReader(encoded), maxResponseBytes)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "sfx response contained no audio"}
}
mimeType := audioResultMIME(contentType, raw)
if mimeType == "" {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("sfx response is not audio (Content-Type %q): %s", contentType, truncateForError(raw))}
}
return &musicgen.Result{Audio: musicgen.Audio{Data: raw, MIME: mimeType}}, nil
}
// audioResultMIME resolves an audio body's MIME type: the response
// Content-Type when it is a concrete audio type, else content sniffing
// (RIFF/WAVE and friends), else "" — the caller treats undetectable as an
// upstream error, mirroring videoMIME. Sniffed "audio/wave" is normalized
// to the conventional "audio/wav".
func audioResultMIME(contentType string, data []byte) string {
if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
return mt
}
if mt := http.DetectContentType(data); strings.HasPrefix(mt, "audio/") {
if mt == "audio/wave" {
return "audio/wav"
}
return mt
}
return ""
}
+146
View File
@@ -0,0 +1,146 @@
// stems.go implements audio.StemSeparationProvider against a Demucs shim
// (audioutils) reached through llama-swap's /upstream passthrough (ADR-0024):
//
// POST /upstream/<id>/v1/stems multipart file[,model,two_stems,format]
//
// The response is a ZIP of the separated stems — one entry per stem, entry
// name = stem name, extension = container. Zip transport keeps a 4-stem WAV
// result (hundreds of MB decoded) off the JSON-of-base64 path entirely.
package llamaswap
import (
"archive/zip"
"bytes"
"context"
"fmt"
"io"
"net/http"
"path"
"strings"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
)
// maxStemsResponseBytes caps the /v1/stems zip body: four WAV stems of a
// long song legitimately pass the 64MB JSON cap. Bounded so a buggy
// upstream can't allocate without limit.
const maxStemsResponseBytes = 512 << 20
// maxStemEntryBytes caps ONE decompressed zip entry — the zip-bomb guard.
// A single WAV stem of even a very long song stays far under this.
const maxStemEntryBytes = 256 << 20
// StemSeparatorModel implements audio.StemSeparationProvider. The id selects
// which upstream llama-swap loads.
func (p *Provider) StemSeparatorModel(id string, opts ...audio.StemSeparatorModelOption) (audio.StemSeparator, error) {
if err := p.requireBaseURL(); err != nil {
return nil, err
}
_ = audio.ApplyStemSeparatorModelOptions(opts)
return &stemSeparatorModel{p: p, id: id}, nil
}
type stemSeparatorModel struct {
p *Provider
id string
}
// SeparateStems implements audio.StemSeparator.
func (m *stemSeparatorModel) SeparateStems(ctx context.Context, req audio.StemSeparationRequest, opts ...audio.StemSeparationOption) (*audio.StemSeparationResult, error) {
req = req.Apply(opts...)
if len(req.Audio) == 0 {
return nil, fmt.Errorf("%w: stem separation requires audio bytes", llm.ErrUnsupported)
}
mode := strings.ToLower(strings.TrimSpace(req.Mode))
if mode != "" && mode != "two" && mode != "four" {
return nil, fmt.Errorf("%w: stem mode must be \"two\" or \"four\", got %q", llm.ErrUnsupported, req.Mode)
}
format := strings.ToLower(strings.TrimSpace(req.Format))
if format != "" && format != "mp3" && format != "wav" {
return nil, fmt.Errorf("%w: stem format must be \"mp3\" or \"wav\", got %q", llm.ErrUnsupported, req.Format)
}
upPath, err := upstreamPath(m.id, "/v1/stems")
if err != nil {
return nil, err
}
// Demucs' two-stem mode is "isolate one source vs the rest"; vocals is
// the split this surface promises ("two" = vocals + accompaniment).
twoStems := ""
if mode == "two" {
twoStems = "vocals"
}
body, contentType, err := buildMultipart("build stems form",
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
[]formField{
{"model", req.Model, false},
{"two_stems", twoStems, false},
{"format", format, false},
})
if err != nil {
return nil, err
}
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, upPath, m.id, contentType, body, maxStemsResponseBytes)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems response contained no data"}
}
zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
if err != nil {
// A non-zip 2xx body is a misconfigured upstream (an HTML error page
// behind a proxy, a JSON soft error) — fail loud, quoting a slice.
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("stems response is not a zip (Content-Type %q): %s", respType, truncateForError(raw))}
}
res := &audio.StemSeparationResult{}
for _, f := range zr.File {
if f.FileInfo().IsDir() {
continue
}
data, err := readZipEntry(f)
if err != nil {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
Message: fmt.Sprintf("stems zip entry %q: %v", f.Name, err)}
}
// Entry name → stem name; extension → MIME. Entries may sit in a
// per-model directory ("htdemucs/vocals.mp3"), so use the base name.
base := path.Base(f.Name)
ext := path.Ext(base)
res.Stems = append(res.Stems, audio.Stem{
Name: strings.TrimSuffix(base, ext),
Audio: data,
MIME: stemMIME(ext),
})
}
if len(res.Stems) == 0 {
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems zip contained no stems"}
}
return res, nil
}
// readZipEntry decompresses one entry, bounded by maxStemEntryBytes so a
// zip bomb can't allocate without limit.
func readZipEntry(f *zip.File) ([]byte, error) {
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
data, err := io.ReadAll(io.LimitReader(rc, maxStemEntryBytes+1))
if err != nil {
return nil, err
}
if int64(len(data)) > maxStemEntryBytes {
return nil, fmt.Errorf("decompressed entry exceeds %d bytes", int64(maxStemEntryBytes))
}
return data, nil
}
// stemMIME maps a stem file extension to its MIME type, reusing speechMIME's
// format table ("" and unknown extensions land on the mp3 default — Demucs'
// own default container).
func stemMIME(ext string) string {
return speechMIME("", strings.TrimPrefix(strings.ToLower(ext), "."))
}
@@ -0,0 +1,332 @@
package llamaswap
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
)
// stemsZipFixture builds a Demucs-style stems zip: entries under a per-model
// directory, entry name = stem name, extension = container.
func stemsZipFixture(t *testing.T, entries map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
zw := zip.NewWriter(&buf)
for name, data := range entries {
w, err := zw.Create(name)
if err != nil {
t.Fatalf("zip create: %v", err)
}
if _, err := w.Write([]byte(data)); err != nil {
t.Fatalf("zip write: %v", err)
}
}
if err := zw.Close(); err != nil {
t.Fatalf("zip close: %v", err)
}
return buf.Bytes()
}
func TestSeparateStems(t *testing.T) {
zipBody := stemsZipFixture(t, map[string]string{
"htdemucs/vocals.mp3": "VOX",
"htdemucs/no_vocals.mp3": "ACC",
})
var gotPath, gotTwoStems, gotModel, gotFormat, gotFilename string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
gotTwoStems = r.FormValue("two_stems")
gotModel = r.FormValue("model")
gotFormat = r.FormValue("format")
if _, hdr, err := r.FormFile("file"); err == nil {
gotFilename = hdr.Filename
} else {
t.Errorf("file part: %v", err)
}
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(zipBody)
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ss, err := p.StemSeparatorModel("audioutils")
if err != nil {
t.Fatalf("StemSeparatorModel: %v", err)
}
res, err := ss.SeparateStems(context.Background(),
audio.StemSeparationRequest{Audio: []byte("SONG"), MIME: "audio/mpeg"},
audio.WithStemMode("two"), audio.WithStemModel("htdemucs_ft"), audio.WithStemFormat("mp3"))
if err != nil {
t.Fatalf("SeparateStems: %v", err)
}
if gotPath != "/upstream/audioutils/v1/stems" {
t.Errorf("path = %q", gotPath)
}
if gotTwoStems != "vocals" || gotModel != "htdemucs_ft" || gotFormat != "mp3" || gotFilename != "audio.mp3" {
t.Errorf("two_stems/model/format/filename = %q/%q/%q/%q", gotTwoStems, gotModel, gotFormat, gotFilename)
}
if len(res.Stems) != 2 {
t.Fatalf("stems = %+v", res.Stems)
}
byName := map[string]audio.Stem{}
for _, s := range res.Stems {
byName[s.Name] = s
}
if v := byName["vocals"]; string(v.Audio) != "VOX" || v.MIME != "audio/mpeg" {
t.Errorf("vocals = %+v", v)
}
if a := byName["no_vocals"]; string(a.Audio) != "ACC" || a.MIME != "audio/mpeg" {
t.Errorf("no_vocals = %+v", a)
}
}
func TestSeparateStemsOmitsUnsetFields(t *testing.T) {
zipBody := stemsZipFixture(t, map[string]string{"vocals.wav": "V"})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
for _, k := range []string{"two_stems", "model", "format"} {
if v, ok := r.MultipartForm.Value[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
_, _ = w.Write(zipBody)
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ss, _ := p.StemSeparatorModel("audioutils")
res, err := ss.SeparateStems(context.Background(),
audio.StemSeparationRequest{Audio: []byte("SONG")})
if err != nil {
t.Fatalf("SeparateStems: %v", err)
}
// Top-level entry, wav extension.
if len(res.Stems) != 1 || res.Stems[0].Name != "vocals" || res.Stems[0].MIME != "audio/wav" {
t.Errorf("stems = %+v", res.Stems)
}
}
func TestSeparateStemsRejectsBadArgs(t *testing.T) {
p := New(WithBaseURL("http://unused"))
ss, _ := p.StemSeparatorModel("audioutils")
if _, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("no audio: err = %v, want ErrUnsupported", err)
}
if _, err := ss.SeparateStems(context.Background(),
audio.StemSeparationRequest{Audio: []byte{1}, Mode: "three"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("mode three: err = %v, want ErrUnsupported", err)
}
if _, err := ss.SeparateStems(context.Background(),
audio.StemSeparationRequest{Audio: []byte{1}, Format: "flac"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("format flac: err = %v, want ErrUnsupported", err)
}
}
func TestSeparateStemsRejectsNonZip(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("<html>proxy error page</html>"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ss, _ := p.StemSeparatorModel("audioutils")
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for non-zip body", err)
}
}
func TestSeparateStemsRejectsEmptyZip(t *testing.T) {
zipBody := stemsZipFixture(t, map[string]string{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(zipBody)
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
ss, _ := p.StemSeparatorModel("audioutils")
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for stem-less zip", err)
}
}
// wavFixture is a minimal RIFF/WAVE header so http.DetectContentType sniffs
// audio/wave.
func wavFixture() []byte {
return []byte("RIFF\x24\x00\x00\x00WAVEfmt ")
}
func TestSFXGenerate(t *testing.T) {
var gotPath string
var gotBody map[string]any
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
_ = json.NewDecoder(r.Body).Decode(&gotBody)
w.Header().Set("Content-Type", "audio/wav")
_, _ = w.Write(wavFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, err := p.SFXModel("sfxgen-stableaudio")
if err != nil {
t.Fatalf("SFXModel: %v", err)
}
res, err := sm.Generate(context.Background(),
musicgen.Request{Prompt: "glass shattering", DurationSeconds: 8},
musicgen.WithSteps(50), musicgen.WithCFGScale(7), musicgen.WithSeed(42))
if err != nil {
t.Fatalf("Generate: %v", err)
}
if gotPath != "/upstream/sfxgen-stableaudio/v1/sfx" {
t.Errorf("path = %q", gotPath)
}
want := map[string]any{"prompt": "glass shattering", "seconds": 8.0, "steps": 50.0, "cfg_scale": 7.0, "seed": 42.0}
for k, w := range want {
if gotBody[k] != w {
t.Errorf("%s = %v, want %v", k, gotBody[k], w)
}
}
if res.Audio.MIME != "audio/wav" || len(res.Audio.Data) == 0 {
t.Errorf("audio = %q/%d bytes", res.Audio.MIME, len(res.Audio.Data))
}
}
func TestSFXOmitsDefaults(t *testing.T) {
var gotBody map[string]any
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_ = json.NewDecoder(r.Body).Decode(&gotBody)
// No Content-Type: the RIFF sniff must still label the clip.
w.Header()["Content-Type"] = nil
_, _ = w.Write(wavFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SFXModel("sfxgen-stableaudio")
res, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"})
if err != nil {
t.Fatalf("Generate: %v", err)
}
for _, k := range []string{"seconds", "steps", "cfg_scale", "seed"} {
if v, ok := gotBody[k]; ok {
t.Errorf("unset request sent %q = %v, want omitted", k, v)
}
}
if res.Audio.MIME != "audio/wav" {
t.Errorf("MIME = %q, want sniffed audio/wav", res.Audio.MIME)
}
}
func TestSFXRejectsBadArgs(t *testing.T) {
p := New(WithBaseURL("http://unused"))
sm, _ := p.SFXModel("sfxgen-stableaudio")
if _, err := sm.Generate(context.Background(), musicgen.Request{Prompt: " "}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("empty prompt: err = %v, want ErrUnsupported", err)
}
if _, err := sm.Generate(context.Background(),
musicgen.Request{Prompt: "boom", Lyrics: "la la"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("lyrics: err = %v, want ErrUnsupported", err)
}
if _, err := sm.Generate(context.Background(),
musicgen.Request{Prompt: "boom", Format: "mp3"}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("format mp3: err = %v, want ErrUnsupported", err)
}
}
func TestSFXRejectsNonAudioResponse(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"detail":"queue full"}`))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
sm, _ := p.SFXModel("sfxgen-stableaudio")
_, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for non-audio body", err)
}
}
func TestEnhance(t *testing.T) {
var gotPath, gotFilename string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatalf("parse multipart: %v", err)
}
if _, hdr, err := r.FormFile("file"); err == nil {
gotFilename = hdr.Filename
} else {
t.Errorf("file part: %v", err)
}
w.Header().Set("Content-Type", "audio/wav")
_, _ = w.Write(wavFixture())
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
en, err := p.SpeechEnhancerModel("audioutils")
if err != nil {
t.Fatalf("SpeechEnhancerModel: %v", err)
}
res, err := en.Enhance(context.Background(),
audio.EnhancementRequest{Audio: []byte("NOISY"), MIME: "audio/ogg"})
if err != nil {
t.Fatalf("Enhance: %v", err)
}
if gotPath != "/upstream/audioutils/v1/enhance" {
t.Errorf("path = %q", gotPath)
}
if gotFilename != "audio.ogg" {
t.Errorf("filename = %q", gotFilename)
}
if res.MIME != "audio/wav" || len(res.Audio) == 0 {
t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio))
}
}
func TestEnhanceRejectsEmptyAudio(t *testing.T) {
p := New(WithBaseURL("http://unused"))
en, _ := p.SpeechEnhancerModel("audioutils")
if _, err := en.Enhance(context.Background(), audio.EnhancementRequest{}); !errors.Is(err, llm.ErrUnsupported) {
t.Errorf("err = %v, want ErrUnsupported", err)
}
}
func TestEnhanceRejectsNonAudioResponse(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("<html>oops</html>"))
}))
defer srv.Close()
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
en, _ := p.SpeechEnhancerModel("audioutils")
_, err := en.Enhance(context.Background(), audio.EnhancementRequest{Audio: []byte("NOISY")})
var apiErr *llm.APIError
if !errors.As(err, &apiErr) {
t.Fatalf("err = %v, want APIError for non-audio body", err)
}
}