feat: wave-3 audio surfaces — stems, sfx, speech enhance, voice clone, translate (ADR-0024)
- audio.StemSeparator/StemSeparationProvider: Demucs zip transport via POST /upstream/<id>/v1/stems (Mode two -> two_stems=vocals; model + format fields); bounded zip unpack, entry name -> stem, ext -> MIME. - SFXModel reuses musicgen against the sync /upstream/<id>/v1/sfx route (JSON prompt/seconds/steps/cfg_scale/seed -> WAV); musicgen.Request gains CFGScale. - audio.SpeechEnhancer/SpeechEnhancementProvider: POST /upstream/<id>/v1/enhance -> WAV (result reuses SpeechResult). - SpeechRequest.ReferenceAudio/ReferenceMIME (+WithReferenceAudio): llamaswap switches to the chatterbox clone route POST /upstream/<id>/v1/audio/speech/upload (input + voice_file), wav MIME fallback. - TranscriptionRequest.Translate (+WithTranslate): translate=true form field, language=auto forced when no explicit hint (whisper.cpp default en would skip translation). - httptest contract tests (zip unpack, clone-route switch, translate + auto-language injection); ADR-0024 (index row deferred — MJ-A backfills the ADR index table and parallel edits would conflict). Co-Authored-By: Claude Fable 5 <[email protected]>
This commit is contained in:
@@ -30,6 +30,17 @@ type SpeechRequest struct {
|
||||
|
||||
// Speed is the playback-rate multiplier; 0 = backend default (1.0).
|
||||
Speed float64
|
||||
|
||||
// ReferenceAudio is a short voice sample for zero-shot voice cloning
|
||||
// (chatterbox style): when set, the model speaks Input in the sampled
|
||||
// voice instead of a named Voice. nil = normal synthesis. Backends
|
||||
// without cloning support must reject a reference-carrying request
|
||||
// rather than silently ignoring it.
|
||||
ReferenceAudio []byte
|
||||
|
||||
// ReferenceMIME is the reference audio's MIME type (e.g. "audio/wav");
|
||||
// "" = let the backend sniff it.
|
||||
ReferenceMIME string
|
||||
}
|
||||
|
||||
// SpeechResult is the canonical synthesis result: raw audio bytes plus the
|
||||
@@ -59,6 +70,11 @@ func WithFormat(f string) SpeechOption { return func(r *SpeechRequest) { r.Forma
|
||||
// WithSpeed sets the playback-rate multiplier.
|
||||
func WithSpeed(s float64) SpeechOption { return func(r *SpeechRequest) { r.Speed = s } }
|
||||
|
||||
// WithReferenceAudio provides a voice sample for zero-shot voice cloning.
|
||||
func WithReferenceAudio(data []byte, mime string) SpeechOption {
|
||||
return func(r *SpeechRequest) { r.ReferenceAudio, r.ReferenceMIME = data, mime }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied. Providers
|
||||
// call this once at the top of Speak.
|
||||
func (r SpeechRequest) Apply(opts ...SpeechOption) SpeechRequest {
|
||||
@@ -121,6 +137,13 @@ type TranscriptionRequest struct {
|
||||
|
||||
// Prompt is optional context or vocabulary to bias decoding; "" = none.
|
||||
Prompt string
|
||||
|
||||
// Translate requests an English translation of the speech instead of a
|
||||
// same-language transcript (whisper's translate task). When set and no
|
||||
// Language is given, providers must force source-language auto-detection
|
||||
// — a backend whose default language is "en" would otherwise skip
|
||||
// translation entirely.
|
||||
Translate bool
|
||||
}
|
||||
|
||||
// TranscriptionResult is the canonical transcription result.
|
||||
@@ -145,6 +168,11 @@ func WithPrompt(p string) TranscriptionOption {
|
||||
return func(r *TranscriptionRequest) { r.Prompt = p }
|
||||
}
|
||||
|
||||
// WithTranslate requests an English translation instead of a transcript.
|
||||
func WithTranslate() TranscriptionOption {
|
||||
return func(r *TranscriptionRequest) { r.Translate = true }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r TranscriptionRequest) Apply(opts ...TranscriptionOption) TranscriptionRequest {
|
||||
for _, opt := range opts {
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package audio
|
||||
|
||||
import "context"
|
||||
|
||||
// EnhancementRequest asks a speech-enhancement backend (DeepFilterNet style)
|
||||
// to denoise a recording. Audio is carried as bytes (never a URL), mirroring
|
||||
// TranscriptionRequest (ADR-0024).
|
||||
type EnhancementRequest struct {
|
||||
// Audio is the encoded audio to enhance.
|
||||
Audio []byte
|
||||
|
||||
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
|
||||
// sniff it.
|
||||
MIME string
|
||||
|
||||
// Filename is the multipart filename hint some backends key their format
|
||||
// detection on; "" derives one from MIME or falls back to "audio".
|
||||
Filename string
|
||||
}
|
||||
|
||||
// EnhancementOption mutates an EnhancementRequest before it is sent.
|
||||
// Reserved for future request settings (the reference backend takes no
|
||||
// parameters).
|
||||
type EnhancementOption func(*EnhancementRequest)
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r EnhancementRequest) Apply(opts ...EnhancementOption) EnhancementRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// SpeechEnhancer denoises speech recordings. The result reuses SpeechResult
|
||||
// (audio bytes + MIME + Raw) — enhancement is audio-in/audio-out exactly
|
||||
// like synthesis.
|
||||
type SpeechEnhancer interface {
|
||||
// Enhance returns the denoised audio.
|
||||
Enhance(ctx context.Context, req EnhancementRequest, opts ...EnhancementOption) (*SpeechResult, error)
|
||||
}
|
||||
|
||||
// SpeechEnhancerModelOption configures a SpeechEnhancer at construction time.
|
||||
// Reserved for future per-model settings.
|
||||
type SpeechEnhancerModelOption func(*SpeechEnhancerModelConfig)
|
||||
|
||||
// SpeechEnhancerModelConfig carries per-model construction settings.
|
||||
type SpeechEnhancerModelConfig struct{}
|
||||
|
||||
// ApplySpeechEnhancerModelOptions folds options into a config.
|
||||
func ApplySpeechEnhancerModelOptions(opts []SpeechEnhancerModelOption) SpeechEnhancerModelConfig {
|
||||
var cfg SpeechEnhancerModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SpeechEnhancementProvider mints SpeechEnhancers bound to one backend.
|
||||
type SpeechEnhancementProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// SpeechEnhancerModel returns a SpeechEnhancer bound to the given id
|
||||
// (passed through to the backend verbatim; no catalog validation).
|
||||
SpeechEnhancerModel(id string, opts ...SpeechEnhancerModelOption) (SpeechEnhancer, error)
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package audio
|
||||
|
||||
import "context"
|
||||
|
||||
// StemSeparationRequest asks a source-separation backend (Demucs style) to
|
||||
// split a mix into stems. Audio is carried as bytes (never a URL), mirroring
|
||||
// TranscriptionRequest. Zero values mean "backend default" (ADR-0024).
|
||||
type StemSeparationRequest struct {
|
||||
// Audio is the encoded mix to separate.
|
||||
Audio []byte
|
||||
|
||||
// MIME is the audio MIME type (e.g. "audio/mpeg"); "" = let the backend
|
||||
// sniff it.
|
||||
MIME string
|
||||
|
||||
// Filename is the multipart filename hint some backends key their format
|
||||
// detection on; "" derives one from MIME or falls back to "audio".
|
||||
Filename string
|
||||
|
||||
// Mode selects the split: "two" (vocals + accompaniment) or "four"
|
||||
// (vocals/drums/bass/other); "" = backend default (four).
|
||||
Mode string
|
||||
|
||||
// Model selects the separator's internal network where the backend
|
||||
// offers several (Demucs: "htdemucs", "htdemucs_ft"); "" = backend
|
||||
// default. This is NOT the provider model id — that is fixed when the
|
||||
// StemSeparator is minted (mirrors BackgroundRemovalRequest.Net).
|
||||
Model string
|
||||
|
||||
// Format is the per-stem audio container ("mp3" or "wav");
|
||||
// "" = backend default.
|
||||
Format string
|
||||
}
|
||||
|
||||
// Stem is one separated source.
|
||||
type Stem struct {
|
||||
// Name is the stem's name ("vocals", "drums", "bass", "other",
|
||||
// "no_vocals", ...), taken from the backend's own labelling.
|
||||
Name string
|
||||
|
||||
// Audio is the encoded stem.
|
||||
Audio []byte
|
||||
|
||||
// MIME is the stem's audio MIME type, e.g. "audio/mpeg".
|
||||
MIME string
|
||||
}
|
||||
|
||||
// StemSeparationResult is the canonical separation result.
|
||||
type StemSeparationResult struct {
|
||||
// Stems are the separated sources, in the order the backend returned
|
||||
// them.
|
||||
Stems []Stem
|
||||
}
|
||||
|
||||
// StemSeparationOption mutates a StemSeparationRequest before it is sent.
|
||||
type StemSeparationOption func(*StemSeparationRequest)
|
||||
|
||||
// WithStemMode selects the split ("two" or "four").
|
||||
func WithStemMode(m string) StemSeparationOption {
|
||||
return func(r *StemSeparationRequest) { r.Mode = m }
|
||||
}
|
||||
|
||||
// WithStemModel selects the separator's internal network.
|
||||
func WithStemModel(m string) StemSeparationOption {
|
||||
return func(r *StemSeparationRequest) { r.Model = m }
|
||||
}
|
||||
|
||||
// WithStemFormat sets the per-stem audio container ("mp3", "wav").
|
||||
func WithStemFormat(f string) StemSeparationOption {
|
||||
return func(r *StemSeparationRequest) { r.Format = f }
|
||||
}
|
||||
|
||||
// Apply returns a copy of the request with all options applied.
|
||||
func (r StemSeparationRequest) Apply(opts ...StemSeparationOption) StemSeparationRequest {
|
||||
for _, opt := range opts {
|
||||
opt(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// StemSeparator splits a mix into stems.
|
||||
type StemSeparator interface {
|
||||
// SeparateStems returns the separated sources. Separation is CPU-bound
|
||||
// and slow (minutes for a full song); bound the call with a context
|
||||
// deadline.
|
||||
SeparateStems(ctx context.Context, req StemSeparationRequest, opts ...StemSeparationOption) (*StemSeparationResult, error)
|
||||
}
|
||||
|
||||
// StemSeparatorModelOption configures a StemSeparator at construction time.
|
||||
// Reserved for future per-model settings.
|
||||
type StemSeparatorModelOption func(*StemSeparatorModelConfig)
|
||||
|
||||
// StemSeparatorModelConfig carries per-model construction settings.
|
||||
type StemSeparatorModelConfig struct{}
|
||||
|
||||
// ApplyStemSeparatorModelOptions folds options into a config.
|
||||
func ApplyStemSeparatorModelOptions(opts []StemSeparatorModelOption) StemSeparatorModelConfig {
|
||||
var cfg StemSeparatorModelConfig
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// StemSeparationProvider mints StemSeparators bound to one backend.
|
||||
type StemSeparationProvider interface {
|
||||
// Name is the registry identifier for the provider.
|
||||
Name() string
|
||||
|
||||
// StemSeparatorModel returns a StemSeparator bound to the given id
|
||||
// (passed through to the backend verbatim; no catalog validation).
|
||||
StemSeparatorModel(id string, opts ...StemSeparatorModelOption) (StemSeparator, error)
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
# ADR-0024: Wave-3 audio surfaces (stems, SFX, speech enhance, voice clone, translate)
|
||||
|
||||
Status: Accepted (2026-07-16)
|
||||
|
||||
## Context
|
||||
|
||||
The llama-swap host is gaining wave-3 audio capabilities (spec: mort
|
||||
docs/specs/2026-07-16-llamaswap-wave3.md): Demucs stem separation and
|
||||
DeepFilterNet speech enhancement (both on the CPU `audioutils` shim), Stable
|
||||
Audio Open sound effects (`sfxgen`), plus two upgrades to existing models —
|
||||
chatterbox's stateless voice-clone route and whisper.cpp's per-request
|
||||
translate flag (both verified against the live images 2026-07-16).
|
||||
|
||||
## Decision
|
||||
|
||||
1. **`audio.StemSeparator` / `StemSeparationProvider`** —
|
||||
`StemSeparationRequest{Audio, MIME, Filename, Mode, Model, Format}` →
|
||||
`StemSeparationResult{Stems []Stem{Name, Audio, MIME}}`.
|
||||
- `Mode` is the caller-facing split: `"two"` (vocals + accompaniment,
|
||||
sent as Demucs' `two_stems=vocals`) or `"four"`; `""` = backend
|
||||
default. `Model` selects the Demucs variant (`htdemucs`/`htdemucs_ft`),
|
||||
mirroring `BackgroundRemovalRequest.Net`.
|
||||
- **The wire format is a ZIP** (`POST /upstream/<id>/v1/stems`): four WAV
|
||||
stems would blow any JSON-of-base64 budget. Entry name → stem name,
|
||||
extension → MIME; entries may sit under a per-model directory. Unpacking
|
||||
is bounded per entry (zip-bomb guard) and a non-zip 2xx body fails loud.
|
||||
2. **`SFXModel` reuses `musicgen`** — a sound effect is a short audio clip
|
||||
from a text prompt; only the provider method differs. The sfxgen route
|
||||
(`POST /upstream/<id>/v1/sfx`, JSON `{prompt, seconds, steps?, cfg_scale?,
|
||||
seed?}`) is SYNCHRONOUS (WAV body), unlike ACE-Step's job queue.
|
||||
`musicgen.Request` gains `CFGScale *float64` for it; lyrics and non-wav
|
||||
formats are rejected (the model can't honor them). The ~11s model ceiling
|
||||
is the backend's to enforce.
|
||||
3. **`audio.SpeechEnhancer` / `SpeechEnhancementProvider`** —
|
||||
`EnhancementRequest{Audio, MIME, Filename}` →
|
||||
`POST /upstream/<id>/v1/enhance` → WAV. The result reuses `SpeechResult`;
|
||||
audio-in/audio-out needs no new result type.
|
||||
4. **Voice cloning is a `SpeechRequest` field, not a new surface** —
|
||||
`ReferenceAudio []byte` + `ReferenceMIME`. When set, the llamaswap
|
||||
speech model switches from JSON `/v1/audio/speech` to multipart
|
||||
`POST /upstream/<id>/v1/audio/speech/upload` (fields `input` +
|
||||
`voice_file`; verified live: stateless per-request cloning, no voice
|
||||
library). Voice/format/speed still ride when set; the clone route's MIME
|
||||
fallback is wav (not the JSON route's mp3). Backends without cloning must
|
||||
reject a reference-carrying request rather than silently ignore it.
|
||||
5. **Translation is a `TranscriptionRequest` bool** — `Translate` maps to
|
||||
whisper.cpp's `translate=true` form field (server.cpp:534). Because that
|
||||
server's language DEFAULT is `en` (which silently skips translation), the
|
||||
provider forces `language=auto` when translating without an explicit
|
||||
language hint; an explicit hint wins.
|
||||
6. **Binary success bodies are validated before wrapping** (ADR-0020 rule):
|
||||
sfx/enhance require positive evidence of audio-ness (declared audio/*
|
||||
Content-Type or sniffed RIFF/WAVE), stems require a parseable zip with
|
||||
at least one stem.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `audio` grows from three surfaces to five; the SFX surface adds zero new
|
||||
types (musicgen reuse) — one format→MIME table and one multipart builder
|
||||
keep serving every audio endpoint.
|
||||
- The clone-route switch means one `SpeechModel` can answer over two wire
|
||||
shapes; tests pin both routes so a regression can't silently drop cloning.
|
||||
- `two_stems` is hardwired to vocals: "isolate X vs the rest" for other
|
||||
sources is a backend capability not exposed in v1 (add a field when a
|
||||
caller needs it, not before).
|
||||
@@ -40,6 +40,11 @@ type Request struct {
|
||||
// Steps is the number of inference steps; nil = backend default.
|
||||
Steps *int
|
||||
|
||||
// CFGScale is the classifier-free-guidance scale; nil = backend default.
|
||||
// Architecture-sensitive, so prefer leaving it nil unless the caller
|
||||
// knows the target model. Backends without the knob ignore it.
|
||||
CFGScale *float64
|
||||
|
||||
// Seed fixes the RNG seed for reproducible output; nil = backend
|
||||
// default (random).
|
||||
Seed *int64
|
||||
@@ -73,6 +78,9 @@ func WithFormat(f string) Option { return func(r *Request) { r.Format = f } }
|
||||
// WithSteps overrides the number of inference steps.
|
||||
func WithSteps(n int) Option { return func(r *Request) { r.Steps = &n } }
|
||||
|
||||
// WithCFGScale overrides the classifier-free-guidance scale.
|
||||
func WithCFGScale(s float64) Option { return func(r *Request) { r.CFGScale = &s } }
|
||||
|
||||
// WithSeed fixes the RNG seed.
|
||||
func WithSeed(seed int64) Option { return func(r *Request) { r.Seed = &seed } }
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
@@ -42,7 +43,11 @@ type speechRequest struct {
|
||||
Speed float64 `json:"speed,omitempty"`
|
||||
}
|
||||
|
||||
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech.
|
||||
// Speak implements audio.SpeechModel via POST {base}/v1/audio/speech, or —
|
||||
// when the request carries reference audio for voice cloning — via multipart
|
||||
// POST {base}/upstream/<id>/v1/audio/speech/upload (the chatterbox clone
|
||||
// route; verified live 2026-07-16: stateless per-request cloning, fields
|
||||
// `input` + `voice_file`).
|
||||
func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts ...audio.SpeechOption) (*audio.SpeechResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Input) == "" {
|
||||
@@ -51,6 +56,9 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
|
||||
if req.Speed < 0 {
|
||||
return nil, fmt.Errorf("%w: speech speed must be >= 0, got %g", llm.ErrUnsupported, req.Speed)
|
||||
}
|
||||
if len(req.ReferenceAudio) > 0 {
|
||||
return m.speakWithReference(ctx, req)
|
||||
}
|
||||
wire := speechRequest{
|
||||
Model: m.id,
|
||||
Input: req.Input,
|
||||
@@ -72,6 +80,49 @@ func (m *speechModel) Speak(ctx context.Context, req audio.SpeechRequest, opts .
|
||||
return &audio.SpeechResult{Audio: audioBytes, MIME: speechMIME(contentType, req.Format)}, nil
|
||||
}
|
||||
|
||||
// speakWithReference performs zero-shot voice cloning via the upstream
|
||||
// passthrough clone route. The reference sample rides as the `voice_file`
|
||||
// part and the text as an `input` field; voice/format/speed still go on the
|
||||
// wire when set (upstreams ignore fields they don't understand). The route's
|
||||
// default container is WAV, so MIME resolution falls back to audio/wav (not
|
||||
// the JSON route's mp3) when neither the response header nor the requested
|
||||
// format decides.
|
||||
func (m *speechModel) speakWithReference(ctx context.Context, req audio.SpeechRequest) (*audio.SpeechResult, error) {
|
||||
path, err := upstreamPath(m.id, "/v1/audio/speech/upload")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
speed := ""
|
||||
if req.Speed != 0 {
|
||||
speed = strconv.FormatFloat(req.Speed, 'g', -1, 64)
|
||||
}
|
||||
body, formType, err := buildMultipart("build speech clone form",
|
||||
filePart{field: "voice_file", filename: transcriptionFilename("", req.ReferenceMIME), data: req.ReferenceAudio},
|
||||
[]formField{
|
||||
{"input", req.Input, true},
|
||||
{"voice", req.Voice, false},
|
||||
{"response_format", req.Format, false},
|
||||
{"speed", speed, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
audioBytes, contentType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, formType, body, maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(audioBytes) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "speech clone response contained no audio"}
|
||||
}
|
||||
mimeType := "audio/wav"
|
||||
if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
|
||||
mimeType = mt
|
||||
} else if req.Format != "" {
|
||||
mimeType = speechMIME("", req.Format)
|
||||
}
|
||||
return &audio.SpeechResult{Audio: audioBytes, MIME: mimeType}, nil
|
||||
}
|
||||
|
||||
// speechMIME resolves the result MIME type: the response Content-Type when it
|
||||
// is a concrete audio type, else a mapping from the requested format, else
|
||||
// audio/mpeg (the OpenAI endpoint's default container is mp3).
|
||||
@@ -118,13 +169,26 @@ func (m *transcriptionModel) Transcribe(ctx context.Context, req audio.Transcrip
|
||||
return nil, fmt.Errorf("%w: transcription requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
|
||||
// Translation is a per-request bool form field on whisper.cpp's server
|
||||
// (server.cpp:534, verified 2026-07-16). Its language DEFAULT is "en",
|
||||
// which would make translate a no-op — so an explicit language hint
|
||||
// wins, but an unset one is forced to auto-detection.
|
||||
translate := ""
|
||||
language := req.Language
|
||||
if req.Translate {
|
||||
translate = "true"
|
||||
if language == "" {
|
||||
language = "auto"
|
||||
}
|
||||
}
|
||||
buf, formType, err := buildMultipart("build transcription form",
|
||||
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
|
||||
[]formField{
|
||||
{"model", m.id, true},
|
||||
{"response_format", "json", true},
|
||||
{"language", req.Language, false},
|
||||
{"language", language, false},
|
||||
{"prompt", req.Prompt, false},
|
||||
{"translate", translate, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
)
|
||||
|
||||
func TestSpeakWithReferenceUsesCloneRoute(t *testing.T) {
|
||||
var gotPath, gotInput, gotVoice, gotFilename string
|
||||
var gotRef []byte
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotInput = r.FormValue("input")
|
||||
gotVoice = r.FormValue("voice")
|
||||
f, hdr, err := r.FormFile("voice_file")
|
||||
if err != nil {
|
||||
t.Fatalf("voice_file part: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
gotRef, _ = io.ReadAll(f)
|
||||
gotFilename = hdr.Filename
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
_, _ = w.Write(wavFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SpeechModel("chatterbox")
|
||||
res, err := sm.Speak(context.Background(),
|
||||
audio.SpeechRequest{Input: "hello in my voice", Voice: "narrator"},
|
||||
audio.WithReferenceAudio([]byte("REFWAV"), "audio/wav"))
|
||||
if err != nil {
|
||||
t.Fatalf("Speak: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/chatterbox/v1/audio/speech/upload" {
|
||||
t.Errorf("path = %q, want clone route", gotPath)
|
||||
}
|
||||
if gotInput != "hello in my voice" || gotVoice != "narrator" {
|
||||
t.Errorf("input/voice = %q/%q", gotInput, gotVoice)
|
||||
}
|
||||
if string(gotRef) != "REFWAV" || gotFilename != "audio.wav" {
|
||||
t.Errorf("ref = %q name = %q", gotRef, gotFilename)
|
||||
}
|
||||
if res.MIME != "audio/wav" || len(res.Audio) == 0 {
|
||||
t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpeakWithReferenceDefaultsToWavMIME(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
for _, k := range []string{"voice", "response_format", "speed"} {
|
||||
if v, ok := r.MultipartForm.Value[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
w.Header()["Content-Type"] = nil // no declared type at all
|
||||
_, _ = w.Write([]byte("RAWAUDIO"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SpeechModel("chatterbox")
|
||||
res, err := sm.Speak(context.Background(),
|
||||
audio.SpeechRequest{Input: "hi"},
|
||||
audio.WithReferenceAudio([]byte("REF"), ""))
|
||||
if err != nil {
|
||||
t.Fatalf("Speak: %v", err)
|
||||
}
|
||||
// The clone route's default container is wav, not the JSON route's mp3.
|
||||
if res.MIME != "audio/wav" {
|
||||
t.Errorf("MIME = %q, want audio/wav fallback", res.MIME)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpeakWithoutReferenceKeepsJSONRoute(t *testing.T) {
|
||||
var gotPath, gotContentType string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
gotContentType = r.Header.Get("Content-Type")
|
||||
_, _ = w.Write([]byte("MP3"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SpeechModel("chatterbox")
|
||||
if _, err := sm.Speak(context.Background(), audio.SpeechRequest{Input: "hi"}); err != nil {
|
||||
t.Fatalf("Speak: %v", err)
|
||||
}
|
||||
if gotPath != "/v1/audio/speech" || gotContentType != "application/json" {
|
||||
t.Errorf("path/content-type = %q/%q, want JSON route", gotPath, gotContentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeTranslateForcesAutoLanguage(t *testing.T) {
|
||||
var gotTranslate, gotLanguage string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotTranslate = r.FormValue("translate")
|
||||
gotLanguage = r.FormValue("language")
|
||||
_, _ = w.Write([]byte(`{"text":"hello"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
|
||||
res, err := tm.Transcribe(context.Background(),
|
||||
audio.TranscriptionRequest{Audio: []byte("AUDIO")},
|
||||
audio.WithTranslate())
|
||||
if err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
// whisper.cpp's server default language is "en", which would skip
|
||||
// translation — translate must force auto-detection when no explicit
|
||||
// language hint was given.
|
||||
if gotTranslate != "true" || gotLanguage != "auto" {
|
||||
t.Errorf("translate/language = %q/%q, want true/auto", gotTranslate, gotLanguage)
|
||||
}
|
||||
if res.Text != "hello" {
|
||||
t.Errorf("text = %q", res.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeTranslateKeepsExplicitLanguage(t *testing.T) {
|
||||
var gotTranslate, gotLanguage string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotTranslate = r.FormValue("translate")
|
||||
gotLanguage = r.FormValue("language")
|
||||
_, _ = w.Write([]byte(`{"text":"hello"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
|
||||
if _, err := tm.Transcribe(context.Background(),
|
||||
audio.TranscriptionRequest{Audio: []byte("AUDIO"), Language: "de", Translate: true}); err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
if gotTranslate != "true" || gotLanguage != "de" {
|
||||
t.Errorf("translate/language = %q/%q, want true/de", gotTranslate, gotLanguage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscribeWithoutTranslateOmitsFields(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
for _, k := range []string{"translate", "language"} {
|
||||
if v, ok := r.MultipartForm.Value[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"text":"hi"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
tm, _ := p.TranscriptionModel("whisper-large-v3-turbo")
|
||||
if _, err := tm.Transcribe(context.Background(),
|
||||
audio.TranscriptionRequest{Audio: []byte("AUDIO")}); err != nil {
|
||||
t.Fatalf("Transcribe: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
// enhance.go implements audio.SpeechEnhancementProvider against a
|
||||
// DeepFilterNet shim (audioutils) reached through llama-swap's /upstream
|
||||
// passthrough (ADR-0024):
|
||||
//
|
||||
// POST /upstream/<id>/v1/enhance multipart file -> WAV
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// SpeechEnhancerModel implements audio.SpeechEnhancementProvider. The id
|
||||
// selects which upstream llama-swap loads.
|
||||
func (p *Provider) SpeechEnhancerModel(id string, opts ...audio.SpeechEnhancerModelOption) (audio.SpeechEnhancer, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = audio.ApplySpeechEnhancerModelOptions(opts)
|
||||
return &speechEnhancerModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type speechEnhancerModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// Enhance implements audio.SpeechEnhancer. The endpoint always answers WAV.
|
||||
func (m *speechEnhancerModel) Enhance(ctx context.Context, req audio.EnhancementRequest, opts ...audio.EnhancementOption) (*audio.SpeechResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Audio) == 0 {
|
||||
return nil, fmt.Errorf("%w: speech enhancement requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
path, err := upstreamPath(m.id, "/v1/enhance")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, contentType, err := buildMultipart("build enhance form",
|
||||
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
|
||||
nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, contentType, body, maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "enhance response contained no audio"}
|
||||
}
|
||||
mimeType := audioResultMIME(respType, raw)
|
||||
if mimeType == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("enhance response is not audio (Content-Type %q): %s", respType, truncateForError(raw))}
|
||||
}
|
||||
return &audio.SpeechResult{Audio: raw, MIME: mimeType}, nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
// sfx.go implements a second musicgen.Model factory against a Stable Audio
|
||||
// Open shim (sfxgen) reached through llama-swap's /upstream passthrough
|
||||
// (ADR-0024):
|
||||
//
|
||||
// POST /upstream/<id>/v1/sfx JSON {prompt, seconds?, steps?, cfg_scale?, seed?}
|
||||
//
|
||||
// Unlike the ACE-Step music path (music.go), /v1/sfx is SYNCHRONOUS: the
|
||||
// response body is the finished WAV clip — no job queue, no polling. The
|
||||
// surface reuses the musicgen types (a sound effect is a short audio clip
|
||||
// from a text prompt); only the provider method differs.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
|
||||
)
|
||||
|
||||
// SFXModel returns a musicgen.Model bound to a sync sound-effect backend.
|
||||
// The id selects which upstream llama-swap loads.
|
||||
func (p *Provider) SFXModel(id string, opts ...musicgen.ModelOption) (musicgen.Model, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = musicgen.ApplyModelOptions(opts)
|
||||
return &sfxModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type sfxModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// sfxRequest is the sfxgen shim's /v1/sfx shape. Optional fields stay off
|
||||
// the wire so the model's own defaults apply.
|
||||
type sfxRequest struct {
|
||||
Prompt string `json:"prompt"`
|
||||
Seconds int `json:"seconds,omitempty"`
|
||||
Steps *int `json:"steps,omitempty"`
|
||||
CFGScale *float64 `json:"cfg_scale,omitempty"`
|
||||
Seed *int64 `json:"seed,omitempty"`
|
||||
}
|
||||
|
||||
// Generate implements musicgen.Model. The clip-length ceiling (~11s for
|
||||
// Stable Audio Open Small) is the backend's to enforce, not this client's.
|
||||
func (m *sfxModel) Generate(ctx context.Context, req musicgen.Request, opts ...musicgen.Option) (*musicgen.Result, error) {
|
||||
req = req.Apply(opts...)
|
||||
if strings.TrimSpace(req.Prompt) == "" {
|
||||
return nil, fmt.Errorf("%w: sfx generation requires a prompt", llm.ErrUnsupported)
|
||||
}
|
||||
if req.Lyrics != "" {
|
||||
return nil, fmt.Errorf("%w: sfx generation does not support lyrics", llm.ErrUnsupported)
|
||||
}
|
||||
if req.DurationSeconds < 0 {
|
||||
return nil, fmt.Errorf("%w: duration must be >= 0, got %d", llm.ErrUnsupported, req.DurationSeconds)
|
||||
}
|
||||
if req.Steps != nil && *req.Steps <= 0 {
|
||||
return nil, fmt.Errorf("%w: inference steps must be > 0, got %d", llm.ErrUnsupported, *req.Steps)
|
||||
}
|
||||
// The endpoint emits WAV only; a caller asking for another container
|
||||
// would silently get mislabelled bytes — reject instead.
|
||||
if f := strings.ToLower(strings.TrimSpace(req.Format)); f != "" && f != "wav" {
|
||||
return nil, fmt.Errorf("%w: sfx output is wav only, got format %q", llm.ErrUnsupported, req.Format)
|
||||
}
|
||||
|
||||
path, err := upstreamPath(m.id, "/v1/sfx")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wire := sfxRequest{
|
||||
Prompt: req.Prompt,
|
||||
Seconds: req.DurationSeconds,
|
||||
Steps: req.Steps,
|
||||
CFGScale: req.CFGScale,
|
||||
Seed: req.Seed,
|
||||
}
|
||||
encoded, err := json.Marshal(wire)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("llama-swap: encode sfx request: %w", err)
|
||||
}
|
||||
raw, contentType, err := m.p.doRaw(ctx, http.MethodPost, path, m.id, "application/json", bytes.NewReader(encoded), maxResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "sfx response contained no audio"}
|
||||
}
|
||||
mimeType := audioResultMIME(contentType, raw)
|
||||
if mimeType == "" {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("sfx response is not audio (Content-Type %q): %s", contentType, truncateForError(raw))}
|
||||
}
|
||||
return &musicgen.Result{Audio: musicgen.Audio{Data: raw, MIME: mimeType}}, nil
|
||||
}
|
||||
|
||||
// audioResultMIME resolves an audio body's MIME type: the response
|
||||
// Content-Type when it is a concrete audio type, else content sniffing
|
||||
// (RIFF/WAVE and friends), else "" — the caller treats undetectable as an
|
||||
// upstream error, mirroring videoMIME. Sniffed "audio/wave" is normalized
|
||||
// to the conventional "audio/wav".
|
||||
func audioResultMIME(contentType string, data []byte) string {
|
||||
if mt := mimeFromContentType(contentType, "audio/"); mt != "" {
|
||||
return mt
|
||||
}
|
||||
if mt := http.DetectContentType(data); strings.HasPrefix(mt, "audio/") {
|
||||
if mt == "audio/wave" {
|
||||
return "audio/wav"
|
||||
}
|
||||
return mt
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// stems.go implements audio.StemSeparationProvider against a Demucs shim
|
||||
// (audioutils) reached through llama-swap's /upstream passthrough (ADR-0024):
|
||||
//
|
||||
// POST /upstream/<id>/v1/stems multipart file[,model,two_stems,format]
|
||||
//
|
||||
// The response is a ZIP of the separated stems — one entry per stem, entry
|
||||
// name = stem name, extension = container. Zip transport keeps a 4-stem WAV
|
||||
// result (hundreds of MB decoded) off the JSON-of-base64 path entirely.
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
)
|
||||
|
||||
// maxStemsResponseBytes caps the /v1/stems zip body: four WAV stems of a
|
||||
// long song legitimately pass the 64MB JSON cap. Bounded so a buggy
|
||||
// upstream can't allocate without limit.
|
||||
const maxStemsResponseBytes = 512 << 20
|
||||
|
||||
// maxStemEntryBytes caps ONE decompressed zip entry — the zip-bomb guard.
|
||||
// A single WAV stem of even a very long song stays far under this.
|
||||
const maxStemEntryBytes = 256 << 20
|
||||
|
||||
// StemSeparatorModel implements audio.StemSeparationProvider. The id selects
|
||||
// which upstream llama-swap loads.
|
||||
func (p *Provider) StemSeparatorModel(id string, opts ...audio.StemSeparatorModelOption) (audio.StemSeparator, error) {
|
||||
if err := p.requireBaseURL(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = audio.ApplyStemSeparatorModelOptions(opts)
|
||||
return &stemSeparatorModel{p: p, id: id}, nil
|
||||
}
|
||||
|
||||
type stemSeparatorModel struct {
|
||||
p *Provider
|
||||
id string
|
||||
}
|
||||
|
||||
// SeparateStems implements audio.StemSeparator.
|
||||
func (m *stemSeparatorModel) SeparateStems(ctx context.Context, req audio.StemSeparationRequest, opts ...audio.StemSeparationOption) (*audio.StemSeparationResult, error) {
|
||||
req = req.Apply(opts...)
|
||||
if len(req.Audio) == 0 {
|
||||
return nil, fmt.Errorf("%w: stem separation requires audio bytes", llm.ErrUnsupported)
|
||||
}
|
||||
mode := strings.ToLower(strings.TrimSpace(req.Mode))
|
||||
if mode != "" && mode != "two" && mode != "four" {
|
||||
return nil, fmt.Errorf("%w: stem mode must be \"two\" or \"four\", got %q", llm.ErrUnsupported, req.Mode)
|
||||
}
|
||||
format := strings.ToLower(strings.TrimSpace(req.Format))
|
||||
if format != "" && format != "mp3" && format != "wav" {
|
||||
return nil, fmt.Errorf("%w: stem format must be \"mp3\" or \"wav\", got %q", llm.ErrUnsupported, req.Format)
|
||||
}
|
||||
upPath, err := upstreamPath(m.id, "/v1/stems")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Demucs' two-stem mode is "isolate one source vs the rest"; vocals is
|
||||
// the split this surface promises ("two" = vocals + accompaniment).
|
||||
twoStems := ""
|
||||
if mode == "two" {
|
||||
twoStems = "vocals"
|
||||
}
|
||||
body, contentType, err := buildMultipart("build stems form",
|
||||
filePart{field: "file", filename: transcriptionFilename(req.Filename, req.MIME), data: req.Audio},
|
||||
[]formField{
|
||||
{"model", req.Model, false},
|
||||
{"two_stems", twoStems, false},
|
||||
{"format", format, false},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, respType, err := m.p.doRaw(ctx, http.MethodPost, upPath, m.id, contentType, body, maxStemsResponseBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems response contained no data"}
|
||||
}
|
||||
zr, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
|
||||
if err != nil {
|
||||
// A non-zip 2xx body is a misconfigured upstream (an HTML error page
|
||||
// behind a proxy, a JSON soft error) — fail loud, quoting a slice.
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("stems response is not a zip (Content-Type %q): %s", respType, truncateForError(raw))}
|
||||
}
|
||||
res := &audio.StemSeparationResult{}
|
||||
for _, f := range zr.File {
|
||||
if f.FileInfo().IsDir() {
|
||||
continue
|
||||
}
|
||||
data, err := readZipEntry(f)
|
||||
if err != nil {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id,
|
||||
Message: fmt.Sprintf("stems zip entry %q: %v", f.Name, err)}
|
||||
}
|
||||
// Entry name → stem name; extension → MIME. Entries may sit in a
|
||||
// per-model directory ("htdemucs/vocals.mp3"), so use the base name.
|
||||
base := path.Base(f.Name)
|
||||
ext := path.Ext(base)
|
||||
res.Stems = append(res.Stems, audio.Stem{
|
||||
Name: strings.TrimSuffix(base, ext),
|
||||
Audio: data,
|
||||
MIME: stemMIME(ext),
|
||||
})
|
||||
}
|
||||
if len(res.Stems) == 0 {
|
||||
return nil, &llm.APIError{Provider: m.p.name, Model: m.id, Message: "stems zip contained no stems"}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// readZipEntry decompresses one entry, bounded by maxStemEntryBytes so a
|
||||
// zip bomb can't allocate without limit.
|
||||
func readZipEntry(f *zip.File) ([]byte, error) {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rc.Close()
|
||||
data, err := io.ReadAll(io.LimitReader(rc, maxStemEntryBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > maxStemEntryBytes {
|
||||
return nil, fmt.Errorf("decompressed entry exceeds %d bytes", int64(maxStemEntryBytes))
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// stemMIME maps a stem file extension to its MIME type, reusing speechMIME's
|
||||
// format table ("" and unknown extensions land on the mp3 default — Demucs'
|
||||
// own default container).
|
||||
func stemMIME(ext string) string {
|
||||
return speechMIME("", strings.TrimPrefix(strings.ToLower(ext), "."))
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
package llamaswap
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/audio"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/llm"
|
||||
"gitea.stevedudenhoeffer.com/steve/majordomo/musicgen"
|
||||
)
|
||||
|
||||
// stemsZipFixture builds a Demucs-style stems zip: entries under a per-model
|
||||
// directory, entry name = stem name, extension = container.
|
||||
func stemsZipFixture(t *testing.T, entries map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for name, data := range entries {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatalf("zip create: %v", err)
|
||||
}
|
||||
if _, err := w.Write([]byte(data)); err != nil {
|
||||
t.Fatalf("zip write: %v", err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatalf("zip close: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestSeparateStems(t *testing.T) {
|
||||
zipBody := stemsZipFixture(t, map[string]string{
|
||||
"htdemucs/vocals.mp3": "VOX",
|
||||
"htdemucs/no_vocals.mp3": "ACC",
|
||||
})
|
||||
var gotPath, gotTwoStems, gotModel, gotFormat, gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
gotTwoStems = r.FormValue("two_stems")
|
||||
gotModel = r.FormValue("model")
|
||||
gotFormat = r.FormValue("format")
|
||||
if _, hdr, err := r.FormFile("file"); err == nil {
|
||||
gotFilename = hdr.Filename
|
||||
} else {
|
||||
t.Errorf("file part: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = w.Write(zipBody)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ss, err := p.StemSeparatorModel("audioutils")
|
||||
if err != nil {
|
||||
t.Fatalf("StemSeparatorModel: %v", err)
|
||||
}
|
||||
res, err := ss.SeparateStems(context.Background(),
|
||||
audio.StemSeparationRequest{Audio: []byte("SONG"), MIME: "audio/mpeg"},
|
||||
audio.WithStemMode("two"), audio.WithStemModel("htdemucs_ft"), audio.WithStemFormat("mp3"))
|
||||
if err != nil {
|
||||
t.Fatalf("SeparateStems: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/audioutils/v1/stems" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotTwoStems != "vocals" || gotModel != "htdemucs_ft" || gotFormat != "mp3" || gotFilename != "audio.mp3" {
|
||||
t.Errorf("two_stems/model/format/filename = %q/%q/%q/%q", gotTwoStems, gotModel, gotFormat, gotFilename)
|
||||
}
|
||||
if len(res.Stems) != 2 {
|
||||
t.Fatalf("stems = %+v", res.Stems)
|
||||
}
|
||||
byName := map[string]audio.Stem{}
|
||||
for _, s := range res.Stems {
|
||||
byName[s.Name] = s
|
||||
}
|
||||
if v := byName["vocals"]; string(v.Audio) != "VOX" || v.MIME != "audio/mpeg" {
|
||||
t.Errorf("vocals = %+v", v)
|
||||
}
|
||||
if a := byName["no_vocals"]; string(a.Audio) != "ACC" || a.MIME != "audio/mpeg" {
|
||||
t.Errorf("no_vocals = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateStemsOmitsUnsetFields(t *testing.T) {
|
||||
zipBody := stemsZipFixture(t, map[string]string{"vocals.wav": "V"})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
for _, k := range []string{"two_stems", "model", "format"} {
|
||||
if v, ok := r.MultipartForm.Value[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
_, _ = w.Write(zipBody)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ss, _ := p.StemSeparatorModel("audioutils")
|
||||
res, err := ss.SeparateStems(context.Background(),
|
||||
audio.StemSeparationRequest{Audio: []byte("SONG")})
|
||||
if err != nil {
|
||||
t.Fatalf("SeparateStems: %v", err)
|
||||
}
|
||||
// Top-level entry, wav extension.
|
||||
if len(res.Stems) != 1 || res.Stems[0].Name != "vocals" || res.Stems[0].MIME != "audio/wav" {
|
||||
t.Errorf("stems = %+v", res.Stems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateStemsRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
ss, _ := p.StemSeparatorModel("audioutils")
|
||||
if _, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("no audio: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := ss.SeparateStems(context.Background(),
|
||||
audio.StemSeparationRequest{Audio: []byte{1}, Mode: "three"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("mode three: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := ss.SeparateStems(context.Background(),
|
||||
audio.StemSeparationRequest{Audio: []byte{1}, Format: "flac"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("format flac: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateStemsRejectsNonZip(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte("<html>proxy error page</html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ss, _ := p.StemSeparatorModel("audioutils")
|
||||
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-zip body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeparateStemsRejectsEmptyZip(t *testing.T) {
|
||||
zipBody := stemsZipFixture(t, map[string]string{})
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
_, _ = w.Write(zipBody)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
ss, _ := p.StemSeparatorModel("audioutils")
|
||||
_, err := ss.SeparateStems(context.Background(), audio.StemSeparationRequest{Audio: []byte("SONG")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for stem-less zip", err)
|
||||
}
|
||||
}
|
||||
|
||||
// wavFixture is a minimal RIFF/WAVE header so http.DetectContentType sniffs
|
||||
// audio/wave.
|
||||
func wavFixture() []byte {
|
||||
return []byte("RIFF\x24\x00\x00\x00WAVEfmt ")
|
||||
}
|
||||
|
||||
func TestSFXGenerate(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
_, _ = w.Write(wavFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, err := p.SFXModel("sfxgen-stableaudio")
|
||||
if err != nil {
|
||||
t.Fatalf("SFXModel: %v", err)
|
||||
}
|
||||
res, err := sm.Generate(context.Background(),
|
||||
musicgen.Request{Prompt: "glass shattering", DurationSeconds: 8},
|
||||
musicgen.WithSteps(50), musicgen.WithCFGScale(7), musicgen.WithSeed(42))
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/sfxgen-stableaudio/v1/sfx" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
want := map[string]any{"prompt": "glass shattering", "seconds": 8.0, "steps": 50.0, "cfg_scale": 7.0, "seed": 42.0}
|
||||
for k, w := range want {
|
||||
if gotBody[k] != w {
|
||||
t.Errorf("%s = %v, want %v", k, gotBody[k], w)
|
||||
}
|
||||
}
|
||||
if res.Audio.MIME != "audio/wav" || len(res.Audio.Data) == 0 {
|
||||
t.Errorf("audio = %q/%d bytes", res.Audio.MIME, len(res.Audio.Data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSFXOmitsDefaults(t *testing.T) {
|
||||
var gotBody map[string]any
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
// No Content-Type: the RIFF sniff must still label the clip.
|
||||
w.Header()["Content-Type"] = nil
|
||||
_, _ = w.Write(wavFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SFXModel("sfxgen-stableaudio")
|
||||
res, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"})
|
||||
if err != nil {
|
||||
t.Fatalf("Generate: %v", err)
|
||||
}
|
||||
for _, k := range []string{"seconds", "steps", "cfg_scale", "seed"} {
|
||||
if v, ok := gotBody[k]; ok {
|
||||
t.Errorf("unset request sent %q = %v, want omitted", k, v)
|
||||
}
|
||||
}
|
||||
if res.Audio.MIME != "audio/wav" {
|
||||
t.Errorf("MIME = %q, want sniffed audio/wav", res.Audio.MIME)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSFXRejectsBadArgs(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
sm, _ := p.SFXModel("sfxgen-stableaudio")
|
||||
if _, err := sm.Generate(context.Background(), musicgen.Request{Prompt: " "}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("empty prompt: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := sm.Generate(context.Background(),
|
||||
musicgen.Request{Prompt: "boom", Lyrics: "la la"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("lyrics: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
if _, err := sm.Generate(context.Background(),
|
||||
musicgen.Request{Prompt: "boom", Format: "mp3"}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("format mp3: err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSFXRejectsNonAudioResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"detail":"queue full"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
sm, _ := p.SFXModel("sfxgen-stableaudio")
|
||||
_, err := sm.Generate(context.Background(), musicgen.Request{Prompt: "boom"})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-audio body", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhance(t *testing.T) {
|
||||
var gotPath, gotFilename string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := r.ParseMultipartForm(1 << 20); err != nil {
|
||||
t.Fatalf("parse multipart: %v", err)
|
||||
}
|
||||
if _, hdr, err := r.FormFile("file"); err == nil {
|
||||
gotFilename = hdr.Filename
|
||||
} else {
|
||||
t.Errorf("file part: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "audio/wav")
|
||||
_, _ = w.Write(wavFixture())
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
en, err := p.SpeechEnhancerModel("audioutils")
|
||||
if err != nil {
|
||||
t.Fatalf("SpeechEnhancerModel: %v", err)
|
||||
}
|
||||
res, err := en.Enhance(context.Background(),
|
||||
audio.EnhancementRequest{Audio: []byte("NOISY"), MIME: "audio/ogg"})
|
||||
if err != nil {
|
||||
t.Fatalf("Enhance: %v", err)
|
||||
}
|
||||
if gotPath != "/upstream/audioutils/v1/enhance" {
|
||||
t.Errorf("path = %q", gotPath)
|
||||
}
|
||||
if gotFilename != "audio.ogg" {
|
||||
t.Errorf("filename = %q", gotFilename)
|
||||
}
|
||||
if res.MIME != "audio/wav" || len(res.Audio) == 0 {
|
||||
t.Errorf("result = %q/%d bytes", res.MIME, len(res.Audio))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhanceRejectsEmptyAudio(t *testing.T) {
|
||||
p := New(WithBaseURL("http://unused"))
|
||||
en, _ := p.SpeechEnhancerModel("audioutils")
|
||||
if _, err := en.Enhance(context.Background(), audio.EnhancementRequest{}); !errors.Is(err, llm.ErrUnsupported) {
|
||||
t.Errorf("err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnhanceRejectsNonAudioResponse(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte("<html>oops</html>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
p := New(WithBaseURL(srv.URL), WithHTTPClient(srv.Client()))
|
||||
en, _ := p.SpeechEnhancerModel("audioutils")
|
||||
_, err := en.Enhance(context.Background(), audio.EnhancementRequest{Audio: []byte("NOISY")})
|
||||
var apiErr *llm.APIError
|
||||
if !errors.As(err, &apiErr) {
|
||||
t.Fatalf("err = %v, want APIError for non-audio body", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user